@vellumai/assistant 0.3.4 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (506) hide show
  1. package/Dockerfile +2 -0
  2. package/README.md +88 -2
  3. package/eslint.config.mjs +31 -0
  4. package/package.json +1 -1
  5. package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
  6. package/scripts/ipc/generate-swift.ts +31 -2
  7. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +438 -1
  8. package/src/__tests__/approval-conversation-turn.test.ts +214 -0
  9. package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
  10. package/src/__tests__/approval-message-composer.test.ts +253 -0
  11. package/src/__tests__/browser-manager.test.ts +1 -0
  12. package/src/__tests__/call-conversation-messages.test.ts +130 -0
  13. package/src/__tests__/call-domain.test.ts +12 -2
  14. package/src/__tests__/call-orchestrator.test.ts +799 -249
  15. package/src/__tests__/call-pointer-messages.test.ts +148 -0
  16. package/src/__tests__/call-recovery.test.ts +3 -0
  17. package/src/__tests__/call-routes-http.test.ts +32 -2
  18. package/src/__tests__/call-store.test.ts +3 -0
  19. package/src/__tests__/channel-approval-routes.test.ts +1277 -98
  20. package/src/__tests__/channel-approval.test.ts +37 -0
  21. package/src/__tests__/channel-approvals.test.ts +36 -50
  22. package/src/__tests__/channel-guardian.test.ts +630 -22
  23. package/src/__tests__/channel-readiness-service.test.ts +324 -0
  24. package/src/__tests__/checker.test.ts +14 -7
  25. package/src/__tests__/clarification-resolver.test.ts +44 -24
  26. package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
  27. package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
  28. package/src/__tests__/config-schema.test.ts +14 -8
  29. package/src/__tests__/context-window-manager.test.ts +30 -2
  30. package/src/__tests__/contradiction-checker.test.ts +20 -5
  31. package/src/__tests__/credential-security-invariants.test.ts +7 -2
  32. package/src/__tests__/daemon-lifecycle.test.ts +13 -12
  33. package/src/__tests__/db-migration-rollback.test.ts +752 -0
  34. package/src/__tests__/dictation-mode-detection.test.ts +63 -0
  35. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
  36. package/src/__tests__/entity-search.test.ts +615 -0
  37. package/src/__tests__/fuzzy-match-property.test.ts +5 -5
  38. package/src/__tests__/guardian-action-store.test.ts +123 -0
  39. package/src/__tests__/guardian-action-sweep.test.ts +277 -0
  40. package/src/__tests__/guardian-dispatch.test.ts +389 -0
  41. package/src/__tests__/guardian-question-copy.test.ts +47 -0
  42. package/src/__tests__/handlers-telegram-config.test.ts +4 -2
  43. package/src/__tests__/handlers-twilio-config.test.ts +533 -0
  44. package/src/__tests__/intent-routing.test.ts +2 -0
  45. package/src/__tests__/ipc-snapshot.test.ts +291 -1
  46. package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
  47. package/src/__tests__/messaging-send-tool.test.ts +65 -0
  48. package/src/__tests__/model-intents.test.ts +96 -0
  49. package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
  50. package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
  51. package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
  52. package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
  53. package/src/__tests__/provider-error-scenarios.test.ts +621 -0
  54. package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
  55. package/src/__tests__/qdrant-manager.test.ts +27 -20
  56. package/src/__tests__/relay-server.test.ts +779 -40
  57. package/src/__tests__/run-orchestrator-assistant-events.test.ts +6 -0
  58. package/src/__tests__/run-orchestrator.test.ts +42 -4
  59. package/src/__tests__/runtime-runs-http.test.ts +17 -1
  60. package/src/__tests__/runtime-runs.test.ts +16 -0
  61. package/src/__tests__/schedule-store.test.ts +18 -4
  62. package/src/__tests__/scheduler-recurrence.test.ts +13 -4
  63. package/src/__tests__/session-abort-tool-results.test.ts +6 -0
  64. package/src/__tests__/session-agent-loop.test.ts +857 -0
  65. package/src/__tests__/session-conflict-gate.test.ts +6 -0
  66. package/src/__tests__/session-pre-run-repair.test.ts +6 -0
  67. package/src/__tests__/session-profile-injection.test.ts +6 -0
  68. package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
  69. package/src/__tests__/session-queue.test.ts +6 -0
  70. package/src/__tests__/session-runtime-assembly.test.ts +321 -13
  71. package/src/__tests__/session-slash-known.test.ts +6 -0
  72. package/src/__tests__/session-slash-queue.test.ts +6 -0
  73. package/src/__tests__/session-slash-unknown.test.ts +6 -0
  74. package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
  75. package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
  76. package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
  77. package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
  78. package/src/__tests__/session-workspace-injection.test.ts +6 -0
  79. package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
  80. package/src/__tests__/skills.test.ts +2 -0
  81. package/src/__tests__/sms-messaging-provider.test.ts +126 -0
  82. package/src/__tests__/starter-task-flow.test.ts +2 -0
  83. package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
  84. package/src/__tests__/system-prompt.test.ts +2 -0
  85. package/src/__tests__/task-management-tools.test.ts +2 -2
  86. package/src/__tests__/task-runner.test.ts +14 -4
  87. package/src/__tests__/terminal-tools.test.ts +25 -19
  88. package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
  89. package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
  90. package/src/__tests__/tool-executor.test.ts +23 -24
  91. package/src/__tests__/trust-store.test.ts +3 -3
  92. package/src/__tests__/twilio-rest.test.ts +29 -0
  93. package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
  94. package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
  95. package/src/__tests__/twilio-routes.test.ts +167 -11
  96. package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
  97. package/src/__tests__/user-reference.test.ts +2 -0
  98. package/src/__tests__/voice-quality.test.ts +222 -0
  99. package/src/__tests__/web-search.test.ts +46 -30
  100. package/src/__tests__/work-item-output.test.ts +110 -0
  101. package/src/agent/loop.ts +1 -1
  102. package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
  103. package/src/amazon/client.ts +1418 -0
  104. package/src/amazon/request-extractor.ts +135 -0
  105. package/src/amazon/session.ts +109 -0
  106. package/src/autonomy/autonomy-store.ts +5 -5
  107. package/src/browser-extension-relay/client.ts +124 -0
  108. package/src/browser-extension-relay/protocol.ts +63 -0
  109. package/src/browser-extension-relay/server.ts +177 -0
  110. package/src/bundler/app-bundler.ts +3 -3
  111. package/src/bundler/bundle-signer.ts +1 -1
  112. package/src/bundler/signature-verifier.ts +1 -1
  113. package/src/calls/call-conversation-messages.ts +33 -0
  114. package/src/calls/call-domain.ts +114 -10
  115. package/src/calls/call-orchestrator.ts +268 -59
  116. package/src/calls/call-pointer-messages.ts +53 -0
  117. package/src/calls/call-recovery.ts +3 -8
  118. package/src/calls/call-store.ts +69 -87
  119. package/src/calls/elevenlabs-config.ts +3 -2
  120. package/src/calls/guardian-action-sweep.ts +105 -0
  121. package/src/calls/guardian-dispatch.ts +203 -0
  122. package/src/calls/guardian-question-copy.ts +133 -0
  123. package/src/calls/relay-server.ts +466 -8
  124. package/src/calls/speaker-identification.ts +1 -1
  125. package/src/calls/twilio-config.ts +22 -14
  126. package/src/calls/twilio-provider.ts +6 -4
  127. package/src/calls/twilio-rest.ts +308 -7
  128. package/src/calls/twilio-routes.ts +65 -12
  129. package/src/calls/types.ts +3 -1
  130. package/src/channels/types.ts +25 -0
  131. package/src/cli/amazon.ts +815 -0
  132. package/src/cli/config-commands.ts +2 -2
  133. package/src/cli/core-commands.ts +4 -3
  134. package/src/cli/influencer.ts +244 -0
  135. package/src/cli/map.ts +89 -6
  136. package/src/cli.ts +1 -1
  137. package/src/config/agent-schema.ts +171 -0
  138. package/src/config/bundled-skills/amazon/SKILL.md +127 -0
  139. package/src/config/bundled-skills/amazon/icon.svg +13 -0
  140. package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
  141. package/src/config/bundled-skills/browser/SKILL.md +1 -0
  142. package/src/config/bundled-skills/browser/TOOLS.json +17 -0
  143. package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
  144. package/src/config/bundled-skills/doordash/SKILL.md +51 -51
  145. package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
  146. package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
  147. package/src/config/bundled-skills/influencer/SKILL.md +144 -0
  148. package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
  149. package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
  150. package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
  151. package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
  152. package/src/config/bundled-skills/media-processing/SKILL.md +176 -0
  153. package/src/config/bundled-skills/media-processing/TOOLS.json +230 -0
  154. package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
  155. package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
  156. package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
  157. package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
  158. package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
  159. package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
  160. package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
  161. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +259 -0
  162. package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
  163. package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +136 -0
  164. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +59 -0
  165. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
  166. package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
  167. package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +143 -0
  168. package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
  169. package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +65 -0
  170. package/src/config/bundled-skills/messaging/SKILL.md +33 -8
  171. package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
  172. package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
  173. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
  174. package/src/config/bundled-skills/phone-calls/SKILL.md +88 -23
  175. package/src/config/bundled-skills/twitter/SKILL.md +19 -3
  176. package/src/config/bundled-skills/twitter/icon.svg +14 -0
  177. package/src/config/bundled-tool-registry.ts +310 -0
  178. package/src/config/calls-schema.ts +181 -0
  179. package/src/config/core-schema.ts +309 -0
  180. package/src/config/defaults.ts +28 -3
  181. package/src/config/env-registry.ts +162 -0
  182. package/src/config/env.ts +175 -0
  183. package/src/config/loader.ts +6 -6
  184. package/src/config/memory-schema.ts +528 -0
  185. package/src/config/sandbox-schema.ts +55 -0
  186. package/src/config/schema.ts +158 -1133
  187. package/src/config/skill-state.ts +1 -1
  188. package/src/config/skills-schema.ts +32 -0
  189. package/src/config/skills.ts +35 -24
  190. package/src/config/system-prompt.ts +131 -56
  191. package/src/config/templates/IDENTITY.md +2 -2
  192. package/src/config/templates/SOUL.md +1 -1
  193. package/src/config/types.ts +1 -0
  194. package/src/config/user-reference.ts +4 -9
  195. package/src/config/vellum-skills/catalog.json +6 -7
  196. package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
  197. package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +4 -3
  198. package/src/config/vellum-skills/sms-setup/SKILL.md +216 -0
  199. package/src/config/vellum-skills/twilio-setup/SKILL.md +40 -8
  200. package/src/context/window-manager.ts +27 -7
  201. package/src/daemon/approval-generators.ts +186 -0
  202. package/src/daemon/approved-devices-store.ts +140 -0
  203. package/src/daemon/assistant-attachments.ts +1 -1
  204. package/src/daemon/classifier.ts +35 -32
  205. package/src/daemon/config-watcher.ts +1 -1
  206. package/src/daemon/daemon-control.ts +217 -0
  207. package/src/daemon/handlers/apps.ts +2 -3
  208. package/src/daemon/handlers/config-channels.ts +158 -0
  209. package/src/daemon/handlers/config-inbox.ts +540 -0
  210. package/src/daemon/handlers/config-ingress.ts +231 -0
  211. package/src/daemon/handlers/config-integrations.ts +258 -0
  212. package/src/daemon/handlers/config-model.ts +143 -0
  213. package/src/daemon/handlers/config-parental.ts +163 -0
  214. package/src/daemon/handlers/config-scheduling.ts +172 -0
  215. package/src/daemon/handlers/config-slack.ts +92 -0
  216. package/src/daemon/handlers/config-telegram.ts +301 -0
  217. package/src/daemon/handlers/config-tools.ts +177 -0
  218. package/src/daemon/handlers/config-trust.ts +104 -0
  219. package/src/daemon/handlers/config-twilio.ts +1080 -0
  220. package/src/daemon/handlers/config.ts +53 -1689
  221. package/src/daemon/handlers/diagnostics.ts +1 -1
  222. package/src/daemon/handlers/dictation.ts +180 -0
  223. package/src/daemon/handlers/documents.ts +18 -32
  224. package/src/daemon/handlers/identity.ts +14 -23
  225. package/src/daemon/handlers/index.ts +11 -0
  226. package/src/daemon/handlers/misc.ts +3 -5
  227. package/src/daemon/handlers/pairing.ts +98 -0
  228. package/src/daemon/handlers/sessions.ts +56 -5
  229. package/src/daemon/handlers/shared.ts +6 -1
  230. package/src/daemon/handlers/skills.ts +1 -1
  231. package/src/daemon/handlers/twitter-auth.ts +2 -0
  232. package/src/daemon/handlers/work-items.ts +17 -9
  233. package/src/daemon/handlers/workspace-files.ts +4 -3
  234. package/src/daemon/install-cli-launchers.ts +113 -0
  235. package/src/daemon/ipc-contract/apps.ts +356 -0
  236. package/src/daemon/ipc-contract/browser.ts +74 -0
  237. package/src/daemon/ipc-contract/computer-use.ts +151 -0
  238. package/src/daemon/ipc-contract/diagnostics.ts +56 -0
  239. package/src/daemon/ipc-contract/documents.ts +74 -0
  240. package/src/daemon/ipc-contract/inbox.ts +209 -0
  241. package/src/daemon/ipc-contract/integrations.ts +284 -0
  242. package/src/daemon/ipc-contract/memory.ts +48 -0
  243. package/src/daemon/ipc-contract/messages.ts +211 -0
  244. package/src/daemon/ipc-contract/pairing.ts +45 -0
  245. package/src/daemon/ipc-contract/parental-control.ts +95 -0
  246. package/src/daemon/ipc-contract/schedules.ts +97 -0
  247. package/src/daemon/ipc-contract/sessions.ts +315 -0
  248. package/src/daemon/ipc-contract/shared.ts +42 -0
  249. package/src/daemon/ipc-contract/skills.ts +120 -0
  250. package/src/daemon/ipc-contract/subagents.ts +58 -0
  251. package/src/daemon/ipc-contract/surfaces.ts +250 -0
  252. package/src/daemon/ipc-contract/trust.ts +60 -0
  253. package/src/daemon/ipc-contract/work-items.ts +225 -0
  254. package/src/daemon/ipc-contract/workspace.ts +113 -0
  255. package/src/daemon/ipc-contract-inventory.json +70 -0
  256. package/src/daemon/ipc-contract-inventory.ts +55 -29
  257. package/src/daemon/ipc-contract.ts +229 -2426
  258. package/src/daemon/ipc-protocol.ts +1 -1
  259. package/src/daemon/ipc-validate.ts +7 -0
  260. package/src/daemon/lifecycle.ts +97 -377
  261. package/src/daemon/pairing-store.ts +177 -0
  262. package/src/daemon/providers-setup.ts +43 -0
  263. package/src/daemon/ride-shotgun-handler.ts +68 -3
  264. package/src/daemon/server.ts +66 -46
  265. package/src/daemon/session-agent-loop-handlers.ts +421 -0
  266. package/src/daemon/session-agent-loop.ts +117 -275
  267. package/src/daemon/session-dynamic-profile.ts +1 -1
  268. package/src/daemon/session-history.ts +1 -1
  269. package/src/daemon/session-media-retry.ts +1 -1
  270. package/src/daemon/session-messaging.ts +37 -2
  271. package/src/daemon/session-notifiers.ts +5 -25
  272. package/src/daemon/session-process.ts +99 -59
  273. package/src/daemon/session-queue-manager.ts +96 -4
  274. package/src/daemon/session-runtime-assembly.ts +199 -10
  275. package/src/daemon/session-surfaces.ts +19 -4
  276. package/src/daemon/session-tool-setup.ts +30 -30
  277. package/src/daemon/session-workspace.ts +1 -1
  278. package/src/daemon/session.ts +35 -2
  279. package/src/daemon/shutdown-handlers.ts +122 -0
  280. package/src/daemon/trace-emitter.ts +1 -1
  281. package/src/daemon/watch-handler.ts +36 -33
  282. package/src/doordash/cart-queries.ts +787 -0
  283. package/src/doordash/client.ts +144 -127
  284. package/src/doordash/order-queries.ts +85 -0
  285. package/src/doordash/queries.ts +10 -1308
  286. package/src/doordash/search-queries.ts +203 -0
  287. package/src/doordash/session.ts +3 -2
  288. package/src/doordash/store-queries.ts +246 -0
  289. package/src/doordash/types.ts +367 -0
  290. package/src/email/providers/agentmail.ts +2 -1
  291. package/src/email/providers/index.ts +3 -2
  292. package/src/email/service.ts +3 -2
  293. package/src/errors.ts +43 -0
  294. package/src/home-base/prebuilt/seed.ts +1 -1
  295. package/src/hooks/cli.ts +6 -5
  296. package/src/hooks/config.ts +6 -8
  297. package/src/hooks/discovery.ts +6 -5
  298. package/src/hooks/manager.ts +4 -3
  299. package/src/hooks/runner.ts +2 -2
  300. package/src/hooks/templates.ts +5 -5
  301. package/src/inbound/public-ingress-urls.ts +6 -4
  302. package/src/index.ts +4 -2
  303. package/src/influencer/client.ts +1104 -0
  304. package/src/instrument.ts +4 -3
  305. package/src/logfire.ts +4 -3
  306. package/src/memory/admin.ts +25 -35
  307. package/src/memory/attachments-store.ts +4 -7
  308. package/src/memory/channel-delivery-store.ts +30 -1
  309. package/src/memory/channel-guardian-store.ts +202 -2
  310. package/src/memory/clarification-resolver.ts +37 -33
  311. package/src/memory/conflict-store.ts +67 -61
  312. package/src/memory/contradiction-checker.ts +141 -117
  313. package/src/memory/conversation-store.ts +335 -51
  314. package/src/memory/db-connection.ts +27 -4
  315. package/src/memory/db-init.ts +265 -4
  316. package/src/memory/db.ts +14 -1
  317. package/src/memory/embedding-backend.ts +27 -5
  318. package/src/memory/embedding-ollama.ts +2 -1
  319. package/src/memory/entity-extractor.ts +38 -35
  320. package/src/memory/guardian-action-store.ts +430 -0
  321. package/src/memory/inbox-escalation-projection.ts +59 -0
  322. package/src/memory/inbox-thread-store.ts +218 -0
  323. package/src/memory/ingress-invite-store.ts +338 -0
  324. package/src/memory/ingress-member-store.ts +350 -0
  325. package/src/memory/items-extractor.ts +91 -97
  326. package/src/memory/job-handlers/index-maintenance.ts +3 -3
  327. package/src/memory/job-handlers/media-processing.ts +69 -0
  328. package/src/memory/job-handlers/summarization.ts +32 -26
  329. package/src/memory/job-utils.ts +3 -10
  330. package/src/memory/jobs-store.ts +8 -10
  331. package/src/memory/jobs-worker.ts +55 -36
  332. package/src/memory/media-store.ts +759 -0
  333. package/src/memory/migrations/001-job-deferrals.ts +45 -0
  334. package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
  335. package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
  336. package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
  337. package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
  338. package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
  339. package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
  340. package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
  341. package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
  342. package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
  343. package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
  344. package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
  345. package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
  346. package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
  347. package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
  348. package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
  349. package/src/memory/migrations/017-memory-items-indexes.ts +10 -0
  350. package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
  351. package/src/memory/migrations/index.ts +24 -0
  352. package/src/memory/migrations/registry.ts +79 -0
  353. package/src/memory/migrations/validate-migration-state.ts +69 -0
  354. package/src/memory/qdrant-manager.ts +49 -8
  355. package/src/memory/query-builder.ts +1 -1
  356. package/src/memory/raw-query.ts +119 -0
  357. package/src/memory/recall-cache.ts +4 -1
  358. package/src/memory/retriever.ts +165 -47
  359. package/src/memory/schema-migration.ts +25 -984
  360. package/src/memory/schema.ts +228 -7
  361. package/src/memory/search/entity.ts +205 -31
  362. package/src/memory/search/lexical.ts +81 -52
  363. package/src/memory/search/ranking.ts +27 -23
  364. package/src/memory/search/semantic.ts +157 -19
  365. package/src/memory/search/types.ts +24 -0
  366. package/src/memory/shared-app-links-store.ts +4 -5
  367. package/src/memory/validation.ts +19 -0
  368. package/src/messaging/draft-store.ts +5 -6
  369. package/src/messaging/provider-types.ts +2 -0
  370. package/src/messaging/providers/sms/adapter.ts +201 -0
  371. package/src/messaging/providers/sms/client.ts +93 -0
  372. package/src/messaging/providers/sms/types.ts +7 -0
  373. package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
  374. package/src/messaging/providers/whatsapp/adapter.ts +136 -0
  375. package/src/messaging/providers/whatsapp/client.ts +67 -0
  376. package/src/messaging/style-analyzer.ts +5 -4
  377. package/src/messaging/thread-summarizer.ts +61 -69
  378. package/src/messaging/triage-engine.ts +62 -71
  379. package/src/migrations/config-merge.ts +53 -0
  380. package/src/migrations/data-layout.ts +68 -0
  381. package/src/migrations/data-merge.ts +33 -0
  382. package/src/migrations/hooks-merge.ts +90 -0
  383. package/src/migrations/index.ts +6 -0
  384. package/src/migrations/log.ts +23 -0
  385. package/src/migrations/skills-merge.ts +33 -0
  386. package/src/migrations/workspace-layout.ts +79 -0
  387. package/src/permissions/checker.ts +133 -11
  388. package/src/permissions/prompter.ts +14 -0
  389. package/src/permissions/shell-identity.ts +31 -1
  390. package/src/permissions/trust-store.ts +21 -1
  391. package/src/providers/anthropic/client.ts +4 -4
  392. package/src/providers/failover.ts +2 -2
  393. package/src/providers/model-intents.ts +70 -0
  394. package/src/providers/ollama/client.ts +2 -1
  395. package/src/providers/provider-send-message.ts +176 -0
  396. package/src/providers/registry.ts +71 -30
  397. package/src/providers/retry.ts +35 -1
  398. package/src/providers/types.ts +12 -1
  399. package/src/runtime/approval-conversation-turn.ts +97 -0
  400. package/src/runtime/approval-message-composer.ts +253 -0
  401. package/src/runtime/channel-approval-parser.ts +36 -2
  402. package/src/runtime/channel-approvals.ts +11 -24
  403. package/src/runtime/channel-guardian-service.ts +88 -21
  404. package/src/runtime/channel-readiness-service.ts +418 -0
  405. package/src/runtime/channel-readiness-types.ts +35 -0
  406. package/src/runtime/channel-retry-sweep.ts +184 -0
  407. package/src/runtime/guardian-context-resolver.ts +108 -0
  408. package/src/runtime/http-server.ts +275 -717
  409. package/src/runtime/http-types.ts +59 -3
  410. package/src/runtime/middleware/auth.ts +116 -0
  411. package/src/runtime/middleware/error-handler.ts +33 -0
  412. package/src/runtime/middleware/twilio-validation.ts +127 -0
  413. package/src/runtime/routes/app-routes.ts +1 -1
  414. package/src/runtime/routes/call-routes.ts +51 -7
  415. package/src/runtime/routes/channel-delivery-routes.ts +170 -0
  416. package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
  417. package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
  418. package/src/runtime/routes/channel-route-shared.ts +144 -0
  419. package/src/runtime/routes/channel-routes.ts +32 -1588
  420. package/src/runtime/routes/conversation-routes.ts +50 -7
  421. package/src/runtime/routes/events-routes.ts +2 -2
  422. package/src/runtime/routes/identity-routes.ts +126 -0
  423. package/src/runtime/routes/pairing-routes.ts +143 -0
  424. package/src/runtime/routes/run-routes.ts +15 -1
  425. package/src/runtime/run-orchestrator.ts +86 -35
  426. package/src/schedule/schedule-store.ts +36 -32
  427. package/src/schedule/scheduler.ts +3 -3
  428. package/src/security/encrypted-store.ts +5 -7
  429. package/src/security/oauth2.ts +45 -15
  430. package/src/security/parental-control-store.ts +183 -0
  431. package/src/security/secret-allowlist.ts +4 -3
  432. package/src/security/secret-scanner.ts +5 -5
  433. package/src/security/secure-keys.ts +1 -1
  434. package/src/security/token-manager.ts +3 -2
  435. package/src/services/vercel-deploy.ts +6 -2
  436. package/src/skills/tool-manifest.ts +3 -3
  437. package/src/skills/vellum-catalog-remote.ts +75 -16
  438. package/src/slack/slack-webhook.ts +2 -1
  439. package/src/swarm/orchestrator.ts +92 -1
  440. package/src/swarm/router-planner.ts +6 -9
  441. package/src/swarm/worker-prompts.ts +9 -12
  442. package/src/tasks/task-compiler.ts +19 -28
  443. package/src/tasks/task-runner.ts +1 -1
  444. package/src/tools/assets/materialize.ts +2 -2
  445. package/src/tools/assets/search.ts +15 -14
  446. package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
  447. package/src/tools/browser/auto-navigate.ts +1 -0
  448. package/src/tools/browser/browser-execution.ts +10 -1
  449. package/src/tools/browser/browser-manager.ts +119 -4
  450. package/src/tools/browser/network-recorder.ts +5 -0
  451. package/src/tools/calls/call-start.ts +1 -0
  452. package/src/tools/credentials/broker.ts +11 -2
  453. package/src/tools/credentials/metadata-store.ts +18 -14
  454. package/src/tools/credentials/post-connect-hooks.ts +61 -0
  455. package/src/tools/credentials/vault.ts +49 -23
  456. package/src/tools/execution-target.ts +11 -1
  457. package/src/tools/executor.ts +68 -9
  458. package/src/tools/host-terminal/cli-discover.ts +1 -1
  459. package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
  460. package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
  461. package/src/tools/network/script-proxy/server.ts +1 -1
  462. package/src/tools/network/script-proxy/session-manager.ts +6 -5
  463. package/src/tools/network/web-fetch.ts +18 -2
  464. package/src/tools/network/web-search.ts +8 -4
  465. package/src/tools/reminder/reminder-store.ts +14 -15
  466. package/src/tools/schedule/create.ts +1 -0
  467. package/src/tools/schedule/list.ts +2 -1
  468. package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
  469. package/src/tools/skills/skill-script-runner.ts +24 -9
  470. package/src/tools/skills/skill-tool-factory.ts +1 -0
  471. package/src/tools/tasks/work-item-enqueue.ts +2 -2
  472. package/src/tools/terminal/evaluate-typescript.ts +21 -12
  473. package/src/tools/terminal/parser.ts +50 -0
  474. package/src/tools/types.ts +2 -0
  475. package/src/tools/watcher/delete.ts +6 -0
  476. package/src/tools/weather/service.ts +1 -1
  477. package/src/twitter/client.ts +190 -24
  478. package/src/twitter/router.ts +1 -1
  479. package/src/twitter/session.ts +4 -3
  480. package/src/util/clipboard.ts +1 -1
  481. package/src/util/errors.ts +65 -8
  482. package/src/util/fs.ts +40 -0
  483. package/src/util/json.ts +10 -0
  484. package/src/util/log-redact.ts +189 -0
  485. package/src/util/logger.ts +19 -17
  486. package/src/util/object.ts +3 -0
  487. package/src/util/platform.ts +105 -363
  488. package/src/util/pricing.ts +1 -1
  489. package/src/util/promise-guard.ts +1 -1
  490. package/src/util/retry.ts +19 -0
  491. package/src/util/row-mapper.ts +79 -0
  492. package/src/util/silently.ts +21 -0
  493. package/src/watcher/engine.ts +5 -1
  494. package/src/watcher/provider-types.ts +20 -0
  495. package/src/watcher/providers/github.ts +156 -0
  496. package/src/watcher/providers/gmail.ts +1 -0
  497. package/src/watcher/providers/google-calendar.ts +1 -0
  498. package/src/watcher/providers/linear.ts +460 -0
  499. package/src/watcher/providers/slack.ts +1 -0
  500. package/src/work-items/work-item-runner.ts +1 -1
  501. package/src/workspace/git-service.ts +1 -1
  502. package/src/workspace/provider-commit-message-generator.ts +51 -22
  503. package/src/__tests__/call-bridge.test.ts +0 -517
  504. package/src/__tests__/session-process-bridge.test.ts +0 -244
  505. package/src/calls/call-bridge.ts +0 -168
  506. package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
@@ -20,7 +20,12 @@ Gmail, Slack, and Telegram setup all require a publicly reachable URL for OAuth
20
20
  2. **If it fails because no client_id is found:** The user needs to create Google Cloud OAuth credentials first. Install and load the **google-oauth-setup** skill (which depends on **public-ingress** for the redirect URI):
21
21
  - Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "google-oauth-setup"`.
22
22
  - Then call `skill_load` with `skill: "google-oauth-setup"`.
23
- - Tell the user: *"Gmail isn't connected yet. I've loaded a setup guide that will walk you through creating Google credentials and connecting your account."*
23
+ - Tell the user Gmail isn't connected yet and briefly explain what the setup involves, then use `ui_show` with `surface_type: "confirmation"` to ask for permission to start:
24
+ - **message:** "Ready to set up Gmail?"
25
+ - **detail:** "I'll open a browser where you sign in to Google, then automate everything else — creating a project, enabling APIs, and connecting your account. Takes 2-3 minutes and you can watch in the browser preview panel."
26
+ - **confirmLabel:** "Get Started"
27
+ - **cancelLabel:** "Not Now"
28
+ - If the user confirms, briefly acknowledge (e.g., "Setting up Gmail now...") and proceed with the setup guide. If they decline, acknowledge and let them know they can set it up later.
24
29
  3. **If the user provides a client_id directly in chat:** Call `credential_store` with `action: "oauth2_connect"`, `service: "gmail"`, and `client_id: "<their value>"`. Include `client_secret` too if they provide one. Everything else is auto-filled.
25
30
 
26
31
  ### Slack
@@ -28,7 +33,12 @@ Gmail, Slack, and Telegram setup all require a publicly reachable URL for OAuth
28
33
  2. **If it fails because no client_id is found:** The user needs to create a Slack App first. Install and load the **slack-oauth-setup** skill (which depends on **public-ingress** for the redirect URI):
29
34
  - Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "slack-oauth-setup"`.
30
35
  - Then call `skill_load` with `skill: "slack-oauth-setup"`.
31
- - Tell the user: *"Slack isn't connected yet. I've loaded a setup guide that will walk you through creating a Slack App and connecting your workspace."*
36
+ - Tell the user Slack isn't connected yet and briefly explain what the setup involves, then use `ui_show` with `surface_type: "confirmation"` to ask for permission to start:
37
+ - **message:** "Ready to set up Slack?"
38
+ - **detail:** "I'll walk you through creating a Slack App and connecting your workspace. The process takes a few minutes, and I'll ask for your approval before each step."
39
+ - **confirmLabel:** "Get Started"
40
+ - **cancelLabel:** "Not Now"
41
+ - Wait for the user to confirm before proceeding with the setup guide. If they decline, acknowledge and let them know they can set it up later.
32
42
  3. **If the user provides client_id and client_secret directly in chat:** Call `credential_store` with `action: "oauth2_connect"`, `service: "slack"`, `client_id`, and `client_secret`. Everything else is auto-filled. Note: Slack always requires a client_secret.
33
43
 
34
44
  ### Telegram
@@ -42,14 +52,14 @@ The telegram-setup skill handles: verifying the bot token from @BotFather, gener
42
52
  The telegram-setup skill also includes **guardian verification**, which links your Telegram account as the trusted guardian for the bot.
43
53
 
44
54
  ### SMS (Twilio)
45
- SMS messaging uses Twilio as the telephony provider. Twilio credentials and phone number configuration are shared with the **phone-calls** skill. Load the **twilio-setup** skill to configure Twilio:
46
- - Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "twilio-setup"`.
47
- - Then call `skill_load` with `skill: "twilio-setup"`.
48
- - Tell the user: *"I've loaded a setup guide for Twilio. It will walk you through configuring your Twilio account for SMS and voice calls."*
55
+ SMS messaging uses Twilio as the telephony provider. Twilio credentials and phone number configuration are shared with the **phone-calls** skill. Load the **sms-setup** skill for complete SMS configuration including compliance and testing:
56
+ - Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "sms-setup"`.
57
+ - Then call `skill_load` with `skill: "sms-setup"`.
58
+ - Tell the user: *"I've loaded the SMS setup guide. It will walk you through configuring Twilio, handling compliance requirements, and testing SMS delivery."*
49
59
 
50
- The twilio-setup skill handles: credential storage (Account SID + Auth Token), phone number provisioning or assignment, and public ingress setup. Once Twilio is configured, SMS is available automatically — no additional feature flag is needed. The assistant's Twilio phone number is used for both outbound SMS and voice calls.
60
+ The sms-setup skill handles: Twilio credential storage (Account SID + Auth Token), phone number provisioning or assignment, public ingress setup, SMS compliance verification, and end-to-end test sending. Once SMS is set up, messaging is available automatically — no additional feature flag is needed.
51
61
 
52
- The twilio-setup skill also includes optional **guardian verification** for SMS, which links your phone number as the trusted guardian. This is the same guardian concept used by Telegram — it ensures only verified users can approve sensitive operations via SMS.
62
+ The sms-setup skill also includes optional **guardian verification** for SMS (inherited from twilio-setup), which links your phone number as the trusted guardian.
53
63
 
54
64
  ## Platform Selection
55
65
 
@@ -83,6 +93,21 @@ Telegram is supported as a messaging provider with limited capabilities compared
83
93
  - The bot can only message users or groups that have previously interacted with it (sent `/start` or been added to a group). Bots cannot initiate conversations with arbitrary phone numbers.
84
94
  - Future support for MTProto user-account sessions may lift some of these restrictions.
85
95
 
96
+ ### SMS (Twilio)
97
+ SMS is supported as a messaging provider with limited capabilities. The conversation ID is the recipient's phone number in E.164 format (e.g. `+14155551234`):
98
+
99
+ - **Send**: Send an SMS to a phone number (high risk — requires user approval)
100
+ - **Auth Test**: Verify Twilio credentials and show the configured phone number
101
+
102
+ **Not available** (SMS limitations):
103
+ - List conversations — SMS is stateless; there is no API to enumerate past conversations
104
+ - Read message history — message history is not available through the gateway
105
+ - Search messages — no search API is available for SMS
106
+
107
+ **SMS limits:**
108
+ - Outbound SMS uses the assistant's configured Twilio phone number as the sender. The phone number must be provisioned and assigned via the twilio-setup skill.
109
+ - SMS messages are subject to Twilio's character limits and carrier filtering. Long messages may be split into multiple segments.
110
+
86
111
  ### Slack-specific
87
112
  - **Add Reaction**: Add an emoji reaction to a message
88
113
  - **Leave Channel**: Leave a Slack channel
@@ -7,12 +7,9 @@ import { memoryItems } from '../../../../memory/schema.js';
7
7
  import { enqueueMemoryJob } from '../../../../memory/jobs-store.js';
8
8
  import { extractStylePatterns } from '../../../../messaging/style-analyzer.js';
9
9
  import { truncate } from '../../../../util/truncate.js';
10
+ import { clampUnitInterval } from '../../../../memory/validation.js';
10
11
  import { resolveProvider, withProviderToken, ok, err } from './shared.js';
11
12
 
12
- function clamp(value: number, min: number, max: number): number {
13
- return Math.min(max, Math.max(min, value));
14
- }
15
-
16
13
  function upsertMemoryItem(opts: {
17
14
  kind: string;
18
15
  subject: string;
@@ -35,7 +32,7 @@ function upsertMemoryItem(opts: {
35
32
  .set({
36
33
  statement: opts.statement,
37
34
  status: 'active',
38
- importance: Math.max(existing.importance ?? 0, opts.importance),
35
+ importance: clampUnitInterval(Math.max(existing.importance ?? 0, opts.importance)),
39
36
  lastSeenAt: now,
40
37
  verificationState: 'assistant_inferred',
41
38
  })
@@ -51,7 +48,7 @@ function upsertMemoryItem(opts: {
51
48
  statement: opts.statement,
52
49
  status: 'active',
53
50
  confidence: 0.8,
54
- importance: opts.importance,
51
+ importance: clampUnitInterval(opts.importance),
55
52
  fingerprint,
56
53
  verificationState: 'assistant_inferred',
57
54
  scopeId: opts.scopeId,
@@ -90,7 +87,7 @@ export async function run(input: Record<string, unknown>, context: ToolContext):
90
87
 
91
88
  for (const pattern of result.stylePatterns) {
92
89
  const subject = `${provider.id} writing style: ${pattern.aspect}`;
93
- const importance = clamp(pattern.importance ?? 0.65, 0.55, 0.85);
90
+ const importance = clampUnitInterval(Math.min(0.85, Math.max(0.55, pattern.importance ?? 0.65)));
94
91
  upsertMemoryItem({ kind: 'style', subject, statement: pattern.summary, importance, scopeId });
95
92
  savedCount++;
96
93
  }
@@ -1,7 +1,7 @@
1
1
  import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
2
2
  import { resolveProvider, withProviderToken, ok, err } from './shared.js';
3
3
 
4
- export async function run(input: Record<string, unknown>, _context: ToolContext): Promise<ToolExecutionResult> {
4
+ export async function run(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
5
5
  const platform = input.platform as string | undefined;
6
6
  const conversationId = input.conversation_id as string;
7
7
  const threadId = input.thread_id as string;
@@ -22,6 +22,7 @@ export async function run(input: Record<string, unknown>, _context: ToolContext)
22
22
  return withProviderToken(provider, async (token) => {
23
23
  const result = await provider.sendMessage(token, conversationId, text, {
24
24
  threadId,
25
+ assistantId: context.assistantId,
25
26
  });
26
27
 
27
28
  return ok(`Reply sent (ID: ${result.id}).`);
@@ -1,7 +1,7 @@
1
1
  import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
2
2
  import { resolveProvider, withProviderToken, ok, err } from './shared.js';
3
3
 
4
- export async function run(input: Record<string, unknown>, _context: ToolContext): Promise<ToolExecutionResult> {
4
+ export async function run(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
5
5
  const platform = input.platform as string | undefined;
6
6
  const conversationId = input.conversation_id as string;
7
7
  const text = input.text as string;
@@ -21,8 +21,12 @@ export async function run(input: Record<string, unknown>, _context: ToolContext)
21
21
  const result = await provider.sendMessage(token, conversationId, text, {
22
22
  subject,
23
23
  inReplyTo,
24
+ assistantId: context.assistantId,
24
25
  });
25
26
 
27
+ if (provider.id === 'sms') {
28
+ return ok(`SMS accepted by Twilio (ID: ${result.id}). Note: "accepted" means Twilio received it for delivery — it has not yet been confirmed as delivered to the handset.`);
29
+ }
26
30
  return ok(`Message sent (ID: ${result.id}).`);
27
31
  });
28
32
  } catch (e) {
@@ -1,12 +1,12 @@
1
1
  ---
2
2
  name: "Phone Calls"
3
- description: "Set up Twilio for outgoing phone calls and place AI-powered voice calls on behalf of the user"
3
+ description: "Set up Twilio for AI-powered voice calls both outgoing calls on behalf of the user and incoming calls where the assistant answers as a receptionist"
4
4
  user-invocable: true
5
5
  metadata: {"vellum": {"emoji": "📞", "requires": {"config": ["calls.enabled"]}}}
6
6
  includes: ["public-ingress"]
7
7
  ---
8
8
 
9
- You are helping the user set up and make outgoing phone calls via Twilio. This skill covers the full lifecycle: Twilio account setup, credential storage, public ingress configuration, enabling the calls feature, placing calls, and monitoring live transcripts.
9
+ You are helping the user set up and manage phone calls via Twilio. This skill covers the full lifecycle: Twilio account setup, credential storage, public ingress configuration, enabling the calls feature, placing outbound calls, receiving inbound calls, and monitoring live transcripts.
10
10
 
11
11
  ## Prerequisites — Shared Twilio Setup
12
12
 
@@ -21,7 +21,9 @@ If Twilio is already configured (check `twilio_config` with `action: "get"`), sk
21
21
 
22
22
  ## Overview
23
23
 
24
- The calling system uses Twilio's ConversationRelay to place outbound phone calls. Twilio works out of the box as the default voice provider. Optionally, you can enable ElevenLabs integration for higher-quality, more natural-sounding voices — but this is entirely optional.
24
+ The calling system uses Twilio's ConversationRelay for both **outbound** and **inbound** voice calls. Twilio works out of the box as the default voice provider. Optionally, you can enable ElevenLabs integration for higher-quality, more natural-sounding voices — but this is entirely optional.
25
+
26
+ ### Outbound calls
25
27
 
26
28
  When a call is placed:
27
29
 
@@ -31,6 +33,17 @@ When a call is placed:
31
33
  4. An LLM-driven orchestrator manages the conversation — receiving caller speech (transcribed by Deepgram), generating responses via Claude, and streaming text back for TTS playback
32
34
  5. The transcript is relayed live to the user's conversation thread
33
35
 
36
+ ### Inbound calls
37
+
38
+ When someone dials the assistant's Twilio phone number:
39
+
40
+ 1. Twilio sends a voice webhook to the gateway at `/webhooks/twilio/voice` (no `callSessionId` in the URL)
41
+ 2. The gateway resolves which assistant owns the dialed number via `resolveAssistantByPhoneNumber`, falling back to the standard routing chain (chat_id, user_id, default/reject). Unmapped numbers are rejected with TwiML `<Reject>`.
42
+ 3. The runtime creates a new session keyed by the Twilio CallSid (`createInboundVoiceSession`)
43
+ 4. Twilio opens a ConversationRelay WebSocket. The relay detects the call is inbound when `initiatedFromConversationId == null` and optionally gates the call behind **guardian voice verification** if a pending challenge exists.
44
+ 5. Once verified (or if no challenge is pending), the LLM orchestrator greets the caller in a receptionist style: "Hello, this is [user]'s assistant. How can I help you today?"
45
+ 6. The assistant converses naturally, using ASK_GUARDIAN to consult the user when needed, just like outbound calls.
46
+
34
47
  Three voice quality modes are available:
35
48
  - **`twilio_standard`** (default) — Fully supported. Standard Twilio TTS with Google voices. No extra setup required.
36
49
  - **`twilio_elevenlabs_tts`** — Fully supported. Uses ElevenLabs voices through Twilio ConversationRelay for more natural speech.
@@ -181,6 +194,25 @@ credential_store action=store service=twilio field=user_phone_number value=+1415
181
194
  | `calls.callerIdentity.allowPerCallOverride` | Whether per-call mode selection is allowed | `true` |
182
195
  | `calls.callerIdentity.userNumber` | Optional E.164 phone number for user-number mode (alternative to storing via `credential_store`) | *(empty)* |
183
196
 
197
+ ## DTMF Callee Verification
198
+
199
+ An optional verification step where the callee must enter a numeric code via their phone's keypad (DTMF tones) before the call proceeds. This ensures the intended person has answered the phone.
200
+
201
+ ### How it works
202
+
203
+ 1. When the call connects and DTMF verification is enabled, a random numeric code is generated (length configured by `calls.verification.codeLength`).
204
+ 2. The verification code is shared with the guardian in the initiating conversation so they know what code was issued.
205
+ 3. The AI voice agent speaks the code digit-by-digit to the callee and asks them to enter it on their keypad.
206
+ 4. The callee enters the code via DTMF (phone keypad tones).
207
+ 5. If the code matches, the call proceeds normally. If the code is incorrect, the agent may re-prompt or end the call depending on configuration.
208
+
209
+ ### Configuration
210
+
211
+ | Setting | Description | Default |
212
+ |---|---|---|
213
+ | `calls.verification.enabled` | Enable DTMF callee verification | `false` |
214
+ | `calls.verification.codeLength` | Number of digits in the verification code | `6` |
215
+
184
216
  ## Optional: Higher Quality Voice with ElevenLabs
185
217
 
186
218
  ElevenLabs integration is entirely optional. The standard Twilio-only setup works unchanged — this section is only relevant if you want to improve voice quality.
@@ -264,7 +296,7 @@ To go back to the default voice at any time:
264
296
  vellum config set calls.voice.mode twilio_standard
265
297
  ```
266
298
 
267
- ## Making Calls
299
+ ## Making Outbound Calls
268
300
 
269
301
  Use the `call_start` tool to place outbound calls. Every call requires:
270
302
  - **phone_number**: The number to call in E.164 format (e.g. `+14155551234`)
@@ -319,6 +351,30 @@ On Twilio trial accounts, outbound calls can ONLY be made to **verified numbers*
319
351
  1. Tell the user they need to verify the number at https://console.twilio.com/us1/develop/phone-numbers/manage/verified
320
352
  2. Or upgrade to a paid Twilio account to call any number
321
353
 
354
+ ## Receiving Inbound Calls
355
+
356
+ Once Twilio is configured and the assistant has a phone number, inbound calls work automatically. When someone dials the assistant's number:
357
+
358
+ 1. The gateway resolves the assistant by phone number and forwards to the runtime
359
+ 2. A new voice session is created, keyed by the Twilio CallSid
360
+ 3. The LLM-driven orchestrator answers in receptionist mode — greeting the caller warmly and asking how it can help
361
+ 4. The conversation proceeds naturally, with ASK_GUARDIAN dispatches to consult the user when needed
362
+
363
+ No additional configuration is needed beyond the standard Twilio setup (Steps 1-5 above). As long as `calls.enabled` is `true` and the phone number has been provisioned/assigned, inbound calls are handled automatically.
364
+
365
+ ### Guardian voice verification for inbound calls
366
+
367
+ Optionally, the user can require callers to verify themselves by entering a six-digit code before the call proceeds. This is managed through the **channel guardian verification** system:
368
+
369
+ 1. The user initiates a verification challenge from the desktop UI for the `voice` channel
370
+ 2. A six-digit code is generated and shown to the user
371
+ 3. When the next inbound call arrives, the relay server detects the pending challenge and prompts the caller: "Please enter your six-digit verification code using your keypad, or speak the digits now."
372
+ 4. The caller enters the code via DTMF (keypad) or by speaking the digits
373
+ 5. If the code matches, a guardian binding is created and the call proceeds normally
374
+ 6. If verification fails after 3 attempts, the call ends with "Verification failed. Goodbye."
375
+
376
+ This feature is separate from the outbound DTMF callee verification. It uses the `ChannelGuardianService` challenge system rather than the per-call verification config.
377
+
322
378
  ## Live Call Monitoring
323
379
 
324
380
  ### Showing the live transcript
@@ -343,37 +399,31 @@ By default, always show the live transcript of the call as it happens. When a ca
343
399
 
344
400
  ### Interacting with a live call
345
401
 
346
- During an active call, the user can type messages in the chat thread to interact with the AI voice agent in real time. Messages are automatically routed to the call via the call bridge, which decides how to handle them based on the call's current state:
347
-
348
- #### Mode 1: Answering questions
402
+ During an active call, the user can interact with the AI voice agent via the HTTP API endpoints:
349
403
 
350
- When the AI voice agent encounters something it needs user input for, a **pending question** appears in the chat. The call status changes to `waiting_on_user`.
404
+ #### Answering questions
351
405
 
352
- 1. A **pending question** appears in `call_status` output
353
- 2. Present the question prominently to the user:
406
+ When the AI voice agent encounters something it needs user input for, it dispatches an **ASK_GUARDIAN** request to all configured guardian channels (mac desktop, Telegram, SMS). The call status changes to `waiting_on_user`.
354
407
 
355
- ```
356
- The person on the call asked something the assistant needs your help with:
357
- "They're asking if you'd prefer the smoking or non-smoking section?"
358
- ```
359
-
360
- 3. The user replies directly in the chat — since there is a pending question, the reply is automatically routed as an **answer** to the AI voice agent
361
- 4. The AI voice agent receives the answer and continues the conversation naturally
408
+ 1. The question is delivered simultaneously to every configured channel. The first channel to respond wins (first-response-wins semantics) -- once one channel provides an answer, the other channels receive a "already answered" notice.
409
+ 2. On the mac desktop, a guardian request thread is created with the question. On Telegram/SMS, the question text and a request code are delivered via the gateway.
410
+ 3. If DTMF callee verification is enabled, the callee must enter a verification code before the call proceeds (see the **DTMF Callee Verification** section above).
411
+ 4. The guardian provides an answer through whichever channel they prefer. The answer is routed to the AI voice agent, which continues the conversation naturally.
362
412
 
363
413
  **Important:** Respond to pending questions quickly. There is a consultation timeout (default: 2 minutes). If no answer is provided in time, the AI voice agent will move on.
364
414
 
365
- #### Mode 2: Steering with instructions
415
+ #### Steering with instructions
366
416
 
367
- When there is **no pending question** but the call is still active, any message the user types in the chat is treated as a **steering instruction**. This lets the user proactively guide the call in real time — for example:
417
+ When there is **no pending question** but the call is still active, the user can send steering instructions via the HTTP API (`POST /v1/calls/:id/instruction`) to proactively guide the call in real time — for example:
368
418
 
369
419
  - "Ask them about their cancellation policy too"
370
420
  - "Wrap up the call, we have what we need"
371
421
  - "Switch to asking about weekend availability instead"
372
422
  - "Be more assertive about getting a discount"
373
423
 
374
- The instruction is injected into the AI voice agent's conversation context as high-priority input, and the agent adjusts its behavior accordingly. A confirmation message ("Instruction relayed to active call.") appears in the chat thread.
424
+ The instruction is injected into the AI voice agent's conversation context as high-priority input, and the agent adjusts its behavior accordingly.
375
425
 
376
- **The user does not need to do anything special** just type a message. The system automatically determines whether it should be an answer or an instruction based on whether a question is pending.
426
+ **Note:** Mid-call steering via the desktop chat thread is no longer supported. The desktop thread only receives pointer/status messages about the call. To steer a call, use the HTTP API endpoints directly.
377
427
 
378
428
  ### Call status values
379
429
 
@@ -414,6 +464,7 @@ The `context` field is powerful — use it to give the agent background that hel
414
464
 
415
465
  ### Things the AI voice agent handles well
416
466
 
467
+ **Outbound calls:**
417
468
  - Making reservations and appointments
418
469
  - Checking business hours, availability, or pricing
419
470
  - Confirming or rescheduling existing appointments
@@ -421,6 +472,12 @@ The `context` field is powerful — use it to give the agent background that hel
421
472
  - Simple customer service interactions
422
473
  - Leaving voicemails (it will speak the message if voicemail picks up)
423
474
 
475
+ **Inbound calls:**
476
+ - Answering as a receptionist and routing caller requests to the user via ASK_GUARDIAN
477
+ - Taking messages when the user is unavailable
478
+ - Answering questions the assistant already knows from memory/context
479
+ - Screening calls with guardian voice verification
480
+
424
481
  ### Things to be aware of
425
482
 
426
483
  - Calls have a maximum duration (configurable via `calls.maxDurationSeconds`, default: 1 hour)
@@ -439,7 +496,7 @@ All call-related settings can be managed via `vellum config`:
439
496
  | `calls.maxDurationSeconds` | Maximum call length in seconds | `3600` (1 hour) |
440
497
  | `calls.userConsultTimeoutSeconds` | How long to wait for user answers | `120` (2 min) |
441
498
  | `calls.disclosure.enabled` | Whether the AI announces itself at call start | `true` |
442
- | `calls.disclosure.text` | The disclosure message spoken at call start | `"I should let you know that I'm an AI assistant calling on behalf of my user."` |
499
+ | `calls.disclosure.text` | The disclosure message spoken at call start | `"At the very beginning of the call, introduce yourself as an assistant calling on behalf of my human."` |
443
500
  | `calls.model` | Override LLM model for call orchestration | *(uses default model)* |
444
501
  | `calls.callerIdentity.allowPerCallOverride` | Allow per-call caller identity selection | `true` |
445
502
  | `calls.callerIdentity.userNumber` | E.164 phone number for user-number mode | *(empty)* |
@@ -464,12 +521,20 @@ vellum config set calls.maxDurationSeconds 7200
464
521
  vellum config set calls.disclosure.enabled false
465
522
 
466
523
  # Custom disclosure message
467
- vellum config set calls.disclosure.text "Just so you know, this is an AI assistant calling for my user."
524
+ vellum config set calls.disclosure.text "Just so you know, this is an assistant calling on behalf of my human."
468
525
 
469
526
  # Give more time for user consultation
470
527
  vellum config set calls.userConsultTimeoutSeconds 300
471
528
  ```
472
529
 
530
+ ## Accepted Regressions
531
+
532
+ The following behavioral changes were introduced with the cross-channel guardian architecture (voice-cross-guardian):
533
+
534
+ - **No more mid-call steering via desktop chat.** The call bridge that routed desktop chat messages to the active call has been removed. The desktop chat thread only receives pointer/status messages about the call. To steer a call, use the HTTP API endpoints directly (`POST /v1/calls/:id/instruction`).
535
+ - **No live transcript mirror in the initiating chat.** The initiating desktop conversation no longer receives a real-time mirror of the call transcript. The initiating chat only gets pointer/status messages (call started, call ended, question asked, etc.).
536
+ - **Guardian questions are dispatched cross-channel.** Rather than appearing only in the initiating desktop thread, ASK_GUARDIAN questions are now dispatched to all configured guardian channels (mac desktop, Telegram, SMS) simultaneously. The first channel to respond wins.
537
+
473
538
  ## Troubleshooting
474
539
 
475
540
  ### "Twilio credentials not configured"
@@ -17,7 +17,7 @@ OAuth uses the official X API v2. It is the most reliable connection method and
17
17
 
18
18
  - Supports: **post** and **reply**
19
19
  - Read-only operations (timeline, search, home, bookmarks, notifications, likes, followers, following, media) always use the browser path directly, regardless of the strategy setting.
20
- - Setup: The user connects OAuth credentials through the Settings UI or the `twitter_auth_start` IPC flow.
20
+ - Setup: Collect the OAuth Client ID (and optional Client Secret) from the user in chat using `credential_store` with `action: "prompt"`, then initiate the `twitter_auth_start` IPC flow. See the **First-Use Decision Flow** for the full sequence.
21
21
  - Set the strategy: `vellum x strategy set oauth`
22
22
 
23
23
  ### Browser session (no developer credentials needed)
@@ -45,15 +45,31 @@ When the user triggers a Twitter operation and no strategy has been configured y
45
45
  Look at `oauthConnected`, `browserSessionActive`, `preferredStrategy`, and `strategyConfigured` in the response. If `strategyConfigured` is `false`, the user has not yet chosen a strategy and should be guided through setup.
46
46
 
47
47
  2. **Present both options with trade-offs:**
48
- - **OAuth**: Most reliable and official. Requires X developer app credentials (OAuth Client ID and optional Client Secret). Supports posting and replying. Set up through Settings UI.
48
+ - **OAuth**: Most reliable and official. Requires X developer app credentials (OAuth Client ID and optional Client Secret). Supports posting and replying. Set up right here in the chat.
49
49
  - **Browser session**: Quick to start, no developer credentials needed. Supports all operations including reading timelines and searching. Set up with `vellum x refresh`.
50
50
 
51
51
  3. **Ask the user which they prefer.** Do not choose for them.
52
52
 
53
53
  4. **Execute setup for the chosen path:**
54
- - If OAuth: Guide the user to the Settings UI to connect their X developer credentials, or initiate the `twitter_auth_start` IPC flow.
54
+ - If OAuth: Collect the credentials in-chat using the secure credential prompt, then connect. Follow the **OAuth Setup Sequence** below.
55
55
  - If browser: Run `vellum x refresh` to capture session cookies from Chrome.
56
56
 
57
+ ### OAuth Setup Sequence
58
+
59
+ When the user chooses OAuth, collect their X developer credentials conversationally using the secure UI:
60
+
61
+ 1. **Collect the Client ID securely:**
62
+ Call `credential_store` with `action: "prompt"`, `service: "integration:twitter"`, `field: "oauth_client_id"`, `label: "X (Twitter) OAuth Client ID"`, `description: "Enter the Client ID from your X Developer App"`, and `placeholder: "your-client-id"`.
63
+
64
+ 2. **Collect the Client Secret (if applicable):**
65
+ Ask the user if their X app uses a confidential client (has a Client Secret). If yes, call `credential_store` with `action: "prompt"`, `service: "integration:twitter"`, `field: "oauth_client_secret"`, `label: "X (Twitter) OAuth Client Secret"`, `description: "Enter the Client Secret from your X Developer App (leave blank if using a public client)"`, and `placeholder: "your-client-secret"`.
66
+
67
+ 3. **Initiate the OAuth flow:**
68
+ Send the `twitter_auth_start` IPC message. This opens the X authorization page in the user's browser. Wait for the flow to complete.
69
+
70
+ 4. **Confirm success:**
71
+ Tell the user: "Great, your X account is connected! You can always update these credentials from the Settings page."
72
+
57
73
  5. **Set the preferred strategy:**
58
74
  ```bash
59
75
  vellum x strategy set <oauth|browser|auto>
@@ -0,0 +1,14 @@
1
+ <svg viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg">
2
+ <rect width="16" height="16" fill="#ffffff"/>
3
+ <rect x="2" y="2" width="12" height="12" fill="#000000"/>
4
+ <rect x="4" y="5" width="2" height="1" fill="#ffffff"/>
5
+ <rect x="5" y="4" width="1" height="3" fill="#ffffff"/>
6
+ <rect x="6" y="5" width="2" height="1" fill="#ffffff"/>
7
+ <rect x="10" y="5" width="2" height="1" fill="#ffffff"/>
8
+ <rect x="11" y="4" width="1" height="3" fill="#ffffff"/>
9
+ <rect x="12" y="5" width="2" height="1" fill="#ffffff"/>
10
+ <rect x="4" y="9" width="1" height="2" fill="#ffffff"/>
11
+ <rect x="5" y="8" width="6" height="1" fill="#ffffff"/>
12
+ <rect x="11" y="9" width="1" height="2" fill="#ffffff"/>
13
+ <rect x="5" y="11" width="6" height="1" fill="#ffffff"/>
14
+ </svg>