@vellumai/assistant 0.3.5 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (487) hide show
  1. package/README.md +51 -0
  2. package/eslint.config.mjs +31 -0
  3. package/package.json +1 -1
  4. package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
  5. package/scripts/ipc/generate-swift.ts +18 -2
  6. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +338 -1
  7. package/src/__tests__/approval-conversation-turn.test.ts +214 -0
  8. package/src/__tests__/browser-manager.test.ts +1 -0
  9. package/src/__tests__/call-conversation-messages.test.ts +130 -0
  10. package/src/__tests__/call-orchestrator.test.ts +752 -271
  11. package/src/__tests__/call-pointer-messages.test.ts +148 -0
  12. package/src/__tests__/call-recovery.test.ts +3 -0
  13. package/src/__tests__/call-routes-http.test.ts +5 -0
  14. package/src/__tests__/call-store.test.ts +3 -0
  15. package/src/__tests__/channel-approval-routes.test.ts +1260 -85
  16. package/src/__tests__/channel-approval.test.ts +37 -0
  17. package/src/__tests__/channel-approvals.test.ts +4 -65
  18. package/src/__tests__/channel-guardian.test.ts +556 -0
  19. package/src/__tests__/channel-readiness-service.test.ts +74 -7
  20. package/src/__tests__/checker.test.ts +14 -7
  21. package/src/__tests__/clarification-resolver.test.ts +44 -24
  22. package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
  23. package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
  24. package/src/__tests__/config-schema.test.ts +12 -7
  25. package/src/__tests__/context-window-manager.test.ts +30 -2
  26. package/src/__tests__/contradiction-checker.test.ts +20 -5
  27. package/src/__tests__/credential-security-invariants.test.ts +6 -2
  28. package/src/__tests__/db-migration-rollback.test.ts +752 -0
  29. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
  30. package/src/__tests__/fuzzy-match-property.test.ts +5 -5
  31. package/src/__tests__/guardian-action-store.test.ts +123 -0
  32. package/src/__tests__/guardian-action-sweep.test.ts +277 -0
  33. package/src/__tests__/guardian-dispatch.test.ts +389 -0
  34. package/src/__tests__/guardian-question-copy.test.ts +47 -0
  35. package/src/__tests__/handlers-telegram-config.test.ts +4 -2
  36. package/src/__tests__/handlers-twilio-config.test.ts +126 -0
  37. package/src/__tests__/intent-routing.test.ts +2 -0
  38. package/src/__tests__/ipc-snapshot.test.ts +228 -1
  39. package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
  40. package/src/__tests__/model-intents.test.ts +96 -0
  41. package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
  42. package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
  43. package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
  44. package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
  45. package/src/__tests__/provider-error-scenarios.test.ts +621 -0
  46. package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
  47. package/src/__tests__/qdrant-manager.test.ts +27 -20
  48. package/src/__tests__/relay-server.test.ts +779 -40
  49. package/src/__tests__/run-orchestrator-assistant-events.test.ts +2 -0
  50. package/src/__tests__/run-orchestrator.test.ts +20 -4
  51. package/src/__tests__/runtime-runs-http.test.ts +17 -1
  52. package/src/__tests__/runtime-runs.test.ts +16 -0
  53. package/src/__tests__/schedule-store.test.ts +18 -4
  54. package/src/__tests__/scheduler-recurrence.test.ts +13 -4
  55. package/src/__tests__/session-abort-tool-results.test.ts +6 -0
  56. package/src/__tests__/session-agent-loop.test.ts +857 -0
  57. package/src/__tests__/session-conflict-gate.test.ts +6 -0
  58. package/src/__tests__/session-pre-run-repair.test.ts +6 -0
  59. package/src/__tests__/session-profile-injection.test.ts +6 -0
  60. package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
  61. package/src/__tests__/session-queue.test.ts +6 -0
  62. package/src/__tests__/session-runtime-assembly.test.ts +237 -13
  63. package/src/__tests__/session-slash-known.test.ts +6 -0
  64. package/src/__tests__/session-slash-queue.test.ts +6 -0
  65. package/src/__tests__/session-slash-unknown.test.ts +6 -0
  66. package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
  67. package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
  68. package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
  69. package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
  70. package/src/__tests__/session-workspace-injection.test.ts +6 -0
  71. package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
  72. package/src/__tests__/skills.test.ts +2 -0
  73. package/src/__tests__/sms-messaging-provider.test.ts +2 -1
  74. package/src/__tests__/starter-task-flow.test.ts +2 -0
  75. package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
  76. package/src/__tests__/system-prompt.test.ts +2 -0
  77. package/src/__tests__/task-management-tools.test.ts +2 -2
  78. package/src/__tests__/task-runner.test.ts +14 -4
  79. package/src/__tests__/terminal-tools.test.ts +25 -19
  80. package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
  81. package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
  82. package/src/__tests__/tool-executor.test.ts +23 -24
  83. package/src/__tests__/trust-store.test.ts +3 -3
  84. package/src/__tests__/twilio-rest.test.ts +29 -0
  85. package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
  86. package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
  87. package/src/__tests__/twilio-routes.test.ts +141 -21
  88. package/src/__tests__/user-reference.test.ts +2 -0
  89. package/src/__tests__/voice-quality.test.ts +222 -0
  90. package/src/__tests__/web-search.test.ts +45 -29
  91. package/src/agent/loop.ts +1 -1
  92. package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
  93. package/src/amazon/client.ts +1418 -0
  94. package/src/amazon/request-extractor.ts +135 -0
  95. package/src/amazon/session.ts +109 -0
  96. package/src/autonomy/autonomy-store.ts +5 -5
  97. package/src/browser-extension-relay/client.ts +124 -0
  98. package/src/browser-extension-relay/protocol.ts +63 -0
  99. package/src/browser-extension-relay/server.ts +177 -0
  100. package/src/bundler/app-bundler.ts +3 -3
  101. package/src/bundler/bundle-signer.ts +1 -1
  102. package/src/bundler/signature-verifier.ts +1 -1
  103. package/src/calls/call-conversation-messages.ts +33 -0
  104. package/src/calls/call-domain.ts +106 -5
  105. package/src/calls/call-orchestrator.ts +252 -54
  106. package/src/calls/call-pointer-messages.ts +53 -0
  107. package/src/calls/call-recovery.ts +3 -8
  108. package/src/calls/call-store.ts +69 -87
  109. package/src/calls/elevenlabs-config.ts +3 -2
  110. package/src/calls/guardian-action-sweep.ts +105 -0
  111. package/src/calls/guardian-dispatch.ts +203 -0
  112. package/src/calls/guardian-question-copy.ts +133 -0
  113. package/src/calls/relay-server.ts +466 -8
  114. package/src/calls/speaker-identification.ts +1 -1
  115. package/src/calls/twilio-config.ts +7 -5
  116. package/src/calls/twilio-provider.ts +6 -4
  117. package/src/calls/twilio-rest.ts +40 -15
  118. package/src/calls/twilio-routes.ts +60 -45
  119. package/src/calls/types.ts +3 -1
  120. package/src/channels/types.ts +25 -0
  121. package/src/cli/amazon.ts +815 -0
  122. package/src/cli/config-commands.ts +2 -2
  123. package/src/cli/core-commands.ts +4 -3
  124. package/src/cli/influencer.ts +244 -0
  125. package/src/cli/map.ts +89 -6
  126. package/src/cli.ts +1 -1
  127. package/src/config/agent-schema.ts +171 -0
  128. package/src/config/bundled-skills/amazon/SKILL.md +127 -0
  129. package/src/config/bundled-skills/amazon/icon.svg +13 -0
  130. package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
  131. package/src/config/bundled-skills/browser/SKILL.md +1 -0
  132. package/src/config/bundled-skills/browser/TOOLS.json +17 -0
  133. package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
  134. package/src/config/bundled-skills/doordash/SKILL.md +51 -51
  135. package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
  136. package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
  137. package/src/config/bundled-skills/influencer/SKILL.md +144 -0
  138. package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
  139. package/src/config/bundled-skills/media-processing/SKILL.md +72 -95
  140. package/src/config/bundled-skills/media-processing/TOOLS.json +57 -147
  141. package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
  142. package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
  143. package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
  144. package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
  145. package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
  146. package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
  147. package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
  148. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +7 -9
  149. package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
  150. package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +88 -253
  151. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +22 -153
  152. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -2
  153. package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +28 -51
  154. package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +35 -270
  155. package/src/config/bundled-skills/messaging/SKILL.md +12 -2
  156. package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
  157. package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
  158. package/src/config/bundled-skills/phone-calls/SKILL.md +86 -21
  159. package/src/config/bundled-skills/twitter/icon.svg +14 -0
  160. package/src/config/bundled-tool-registry.ts +310 -0
  161. package/src/config/calls-schema.ts +181 -0
  162. package/src/config/core-schema.ts +309 -0
  163. package/src/config/defaults.ts +27 -3
  164. package/src/config/env-registry.ts +169 -0
  165. package/src/config/env.ts +175 -0
  166. package/src/config/loader.ts +6 -6
  167. package/src/config/memory-schema.ts +528 -0
  168. package/src/config/sandbox-schema.ts +55 -0
  169. package/src/config/schema.ts +157 -1138
  170. package/src/config/skill-state.ts +1 -1
  171. package/src/config/skills-schema.ts +32 -0
  172. package/src/config/skills.ts +35 -24
  173. package/src/config/system-prompt.ts +107 -56
  174. package/src/config/templates/SOUL.md +1 -1
  175. package/src/config/types.ts +1 -0
  176. package/src/config/user-reference.ts +4 -9
  177. package/src/config/vellum-skills/catalog.json +0 -7
  178. package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
  179. package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +1 -0
  180. package/src/config/vellum-skills/sms-setup/SKILL.md +112 -14
  181. package/src/context/window-manager.ts +27 -7
  182. package/src/daemon/approval-generators.ts +186 -0
  183. package/src/daemon/approved-devices-store.ts +140 -0
  184. package/src/daemon/assistant-attachments.ts +1 -1
  185. package/src/daemon/classifier.ts +35 -32
  186. package/src/daemon/config-watcher.ts +1 -1
  187. package/src/daemon/daemon-control.ts +254 -0
  188. package/src/daemon/handlers/apps.ts +2 -3
  189. package/src/daemon/handlers/config-channels.ts +158 -0
  190. package/src/daemon/handlers/config-inbox.ts +540 -0
  191. package/src/daemon/handlers/config-ingress.ts +231 -0
  192. package/src/daemon/handlers/config-integrations.ts +258 -0
  193. package/src/daemon/handlers/config-model.ts +143 -0
  194. package/src/daemon/handlers/config-parental.ts +163 -0
  195. package/src/daemon/handlers/config-scheduling.ts +172 -0
  196. package/src/daemon/handlers/config-slack.ts +92 -0
  197. package/src/daemon/handlers/config-telegram.ts +301 -0
  198. package/src/daemon/handlers/config-tools.ts +177 -0
  199. package/src/daemon/handlers/config-trust.ts +104 -0
  200. package/src/daemon/handlers/config-twilio.ts +1080 -0
  201. package/src/daemon/handlers/config.ts +53 -2463
  202. package/src/daemon/handlers/diagnostics.ts +1 -1
  203. package/src/daemon/handlers/dictation.ts +4 -6
  204. package/src/daemon/handlers/documents.ts +18 -32
  205. package/src/daemon/handlers/index.ts +9 -0
  206. package/src/daemon/handlers/misc.ts +3 -5
  207. package/src/daemon/handlers/pairing.ts +98 -0
  208. package/src/daemon/handlers/sessions.ts +74 -5
  209. package/src/daemon/handlers/shared.ts +3 -1
  210. package/src/daemon/handlers/skills.ts +1 -1
  211. package/src/daemon/handlers/twitter-auth.ts +2 -0
  212. package/src/daemon/handlers/work-items.ts +2 -2
  213. package/src/daemon/handlers/workspace-files.ts +4 -3
  214. package/src/daemon/install-cli-launchers.ts +113 -0
  215. package/src/daemon/ipc-contract/apps.ts +356 -0
  216. package/src/daemon/ipc-contract/browser.ts +74 -0
  217. package/src/daemon/ipc-contract/computer-use.ts +151 -0
  218. package/src/daemon/ipc-contract/diagnostics.ts +56 -0
  219. package/src/daemon/ipc-contract/documents.ts +74 -0
  220. package/src/daemon/ipc-contract/inbox.ts +209 -0
  221. package/src/daemon/ipc-contract/integrations.ts +284 -0
  222. package/src/daemon/ipc-contract/memory.ts +48 -0
  223. package/src/daemon/ipc-contract/messages.ts +211 -0
  224. package/src/daemon/ipc-contract/pairing.ts +45 -0
  225. package/src/daemon/ipc-contract/parental-control.ts +95 -0
  226. package/src/daemon/ipc-contract/schedules.ts +97 -0
  227. package/src/daemon/ipc-contract/sessions.ts +321 -0
  228. package/src/daemon/ipc-contract/shared.ts +42 -0
  229. package/src/daemon/ipc-contract/skills.ts +120 -0
  230. package/src/daemon/ipc-contract/subagents.ts +58 -0
  231. package/src/daemon/ipc-contract/surfaces.ts +250 -0
  232. package/src/daemon/ipc-contract/trust.ts +60 -0
  233. package/src/daemon/ipc-contract/work-items.ts +225 -0
  234. package/src/daemon/ipc-contract/workspace.ts +113 -0
  235. package/src/daemon/ipc-contract-inventory.json +62 -0
  236. package/src/daemon/ipc-contract-inventory.ts +55 -29
  237. package/src/daemon/ipc-contract.ts +227 -2527
  238. package/src/daemon/ipc-protocol.ts +1 -1
  239. package/src/daemon/ipc-validate.ts +7 -0
  240. package/src/daemon/lifecycle.ts +97 -379
  241. package/src/daemon/pairing-store.ts +177 -0
  242. package/src/daemon/providers-setup.ts +43 -0
  243. package/src/daemon/ride-shotgun-handler.ts +67 -2
  244. package/src/daemon/server.ts +60 -44
  245. package/src/daemon/session-agent-loop-handlers.ts +421 -0
  246. package/src/daemon/session-agent-loop.ts +113 -275
  247. package/src/daemon/session-dynamic-profile.ts +1 -1
  248. package/src/daemon/session-history.ts +1 -1
  249. package/src/daemon/session-media-retry.ts +1 -1
  250. package/src/daemon/session-messaging.ts +37 -2
  251. package/src/daemon/session-notifiers.ts +5 -25
  252. package/src/daemon/session-process.ts +99 -59
  253. package/src/daemon/session-queue-manager.ts +98 -4
  254. package/src/daemon/session-runtime-assembly.ts +149 -15
  255. package/src/daemon/session-surfaces.ts +26 -4
  256. package/src/daemon/session-tool-setup.ts +28 -30
  257. package/src/daemon/session-workspace.ts +1 -1
  258. package/src/daemon/session.ts +24 -1
  259. package/src/daemon/shutdown-handlers.ts +122 -0
  260. package/src/daemon/trace-emitter.ts +1 -1
  261. package/src/daemon/watch-handler.ts +36 -33
  262. package/src/doordash/cart-queries.ts +787 -0
  263. package/src/doordash/client.ts +144 -127
  264. package/src/doordash/order-queries.ts +85 -0
  265. package/src/doordash/queries.ts +10 -1308
  266. package/src/doordash/search-queries.ts +203 -0
  267. package/src/doordash/session.ts +3 -2
  268. package/src/doordash/store-queries.ts +246 -0
  269. package/src/doordash/types.ts +367 -0
  270. package/src/email/providers/agentmail.ts +2 -1
  271. package/src/email/providers/index.ts +3 -2
  272. package/src/email/service.ts +3 -2
  273. package/src/errors.ts +43 -0
  274. package/src/home-base/prebuilt/seed.ts +1 -1
  275. package/src/hooks/cli.ts +6 -5
  276. package/src/hooks/config.ts +6 -8
  277. package/src/hooks/discovery.ts +6 -5
  278. package/src/hooks/manager.ts +4 -3
  279. package/src/hooks/runner.ts +2 -2
  280. package/src/hooks/templates.ts +5 -5
  281. package/src/inbound/public-ingress-urls.ts +3 -1
  282. package/src/index.ts +4 -2
  283. package/src/influencer/client.ts +1104 -0
  284. package/src/instrument.ts +4 -3
  285. package/src/logfire.ts +4 -3
  286. package/src/memory/admin.ts +25 -35
  287. package/src/memory/attachments-store.ts +4 -7
  288. package/src/memory/channel-delivery-store.ts +30 -1
  289. package/src/memory/channel-guardian-store.ts +200 -1
  290. package/src/memory/clarification-resolver.ts +37 -33
  291. package/src/memory/conflict-store.ts +67 -61
  292. package/src/memory/contradiction-checker.ts +141 -117
  293. package/src/memory/conversation-store.ts +335 -51
  294. package/src/memory/db-connection.ts +27 -4
  295. package/src/memory/db-init.ts +121 -4
  296. package/src/memory/db.ts +14 -1
  297. package/src/memory/embedding-backend.ts +27 -5
  298. package/src/memory/embedding-ollama.ts +2 -1
  299. package/src/memory/entity-extractor.ts +38 -35
  300. package/src/memory/guardian-action-store.ts +430 -0
  301. package/src/memory/inbox-escalation-projection.ts +59 -0
  302. package/src/memory/inbox-thread-store.ts +218 -0
  303. package/src/memory/ingress-invite-store.ts +338 -0
  304. package/src/memory/ingress-member-store.ts +350 -0
  305. package/src/memory/items-extractor.ts +91 -97
  306. package/src/memory/job-handlers/index-maintenance.ts +3 -3
  307. package/src/memory/job-handlers/media-processing.ts +11 -42
  308. package/src/memory/job-handlers/summarization.ts +32 -26
  309. package/src/memory/job-utils.ts +3 -10
  310. package/src/memory/jobs-store.ts +6 -9
  311. package/src/memory/jobs-worker.ts +51 -36
  312. package/src/memory/migrations/001-job-deferrals.ts +45 -0
  313. package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
  314. package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
  315. package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
  316. package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
  317. package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
  318. package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
  319. package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
  320. package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
  321. package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
  322. package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
  323. package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
  324. package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
  325. package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
  326. package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
  327. package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
  328. package/src/memory/migrations/017-memory-items-indexes.ts +12 -0
  329. package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
  330. package/src/memory/migrations/index.ts +24 -0
  331. package/src/memory/migrations/registry.ts +79 -0
  332. package/src/memory/migrations/validate-migration-state.ts +69 -0
  333. package/src/memory/qdrant-manager.ts +49 -8
  334. package/src/memory/query-builder.ts +1 -1
  335. package/src/memory/raw-query.ts +119 -0
  336. package/src/memory/recall-cache.ts +4 -1
  337. package/src/memory/retriever.ts +163 -47
  338. package/src/memory/schema-migration.ts +25 -984
  339. package/src/memory/schema.ts +130 -7
  340. package/src/memory/search/entity.ts +10 -19
  341. package/src/memory/search/lexical.ts +81 -52
  342. package/src/memory/search/ranking.ts +21 -22
  343. package/src/memory/search/semantic.ts +157 -19
  344. package/src/memory/shared-app-links-store.ts +4 -5
  345. package/src/memory/validation.ts +19 -0
  346. package/src/messaging/draft-store.ts +5 -6
  347. package/src/messaging/providers/sms/adapter.ts +3 -6
  348. package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
  349. package/src/messaging/providers/whatsapp/adapter.ts +136 -0
  350. package/src/messaging/providers/whatsapp/client.ts +67 -0
  351. package/src/messaging/style-analyzer.ts +5 -4
  352. package/src/messaging/thread-summarizer.ts +61 -69
  353. package/src/messaging/triage-engine.ts +62 -71
  354. package/src/migrations/config-merge.ts +53 -0
  355. package/src/migrations/data-layout.ts +68 -0
  356. package/src/migrations/data-merge.ts +33 -0
  357. package/src/migrations/hooks-merge.ts +90 -0
  358. package/src/migrations/index.ts +6 -0
  359. package/src/migrations/log.ts +23 -0
  360. package/src/migrations/skills-merge.ts +33 -0
  361. package/src/migrations/workspace-layout.ts +79 -0
  362. package/src/permissions/checker.ts +126 -11
  363. package/src/permissions/prompter.ts +14 -0
  364. package/src/permissions/shell-identity.ts +31 -1
  365. package/src/permissions/trust-store.ts +21 -1
  366. package/src/providers/anthropic/client.ts +4 -4
  367. package/src/providers/failover.ts +2 -2
  368. package/src/providers/model-intents.ts +70 -0
  369. package/src/providers/ollama/client.ts +2 -1
  370. package/src/providers/provider-send-message.ts +176 -0
  371. package/src/providers/registry.ts +71 -30
  372. package/src/providers/retry.ts +35 -1
  373. package/src/providers/types.ts +12 -1
  374. package/src/runtime/approval-conversation-turn.ts +97 -0
  375. package/src/runtime/approval-message-composer.ts +115 -5
  376. package/src/runtime/assistant-event-hub.ts +3 -1
  377. package/src/runtime/channel-approval-parser.ts +36 -2
  378. package/src/runtime/channel-approvals.ts +0 -21
  379. package/src/runtime/channel-guardian-service.ts +48 -7
  380. package/src/runtime/channel-readiness-service.ts +160 -34
  381. package/src/runtime/channel-readiness-types.ts +10 -4
  382. package/src/runtime/channel-retry-sweep.ts +184 -0
  383. package/src/runtime/guardian-context-resolver.ts +108 -0
  384. package/src/runtime/http-server.ts +289 -745
  385. package/src/runtime/http-types.ts +56 -3
  386. package/src/runtime/middleware/auth.ts +116 -0
  387. package/src/runtime/middleware/error-handler.ts +33 -0
  388. package/src/runtime/middleware/twilio-validation.ts +127 -0
  389. package/src/runtime/routes/app-routes.ts +1 -1
  390. package/src/runtime/routes/call-routes.ts +49 -6
  391. package/src/runtime/routes/channel-delivery-routes.ts +170 -0
  392. package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
  393. package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
  394. package/src/runtime/routes/channel-route-shared.ts +144 -0
  395. package/src/runtime/routes/channel-routes.ts +32 -1634
  396. package/src/runtime/routes/conversation-routes.ts +50 -7
  397. package/src/runtime/routes/events-routes.ts +2 -2
  398. package/src/runtime/routes/identity-routes.ts +126 -0
  399. package/src/runtime/routes/pairing-routes.ts +144 -0
  400. package/src/runtime/routes/run-routes.ts +15 -1
  401. package/src/runtime/run-orchestrator.ts +52 -34
  402. package/src/schedule/schedule-store.ts +36 -32
  403. package/src/schedule/scheduler.ts +3 -3
  404. package/src/security/encrypted-store.ts +5 -7
  405. package/src/security/oauth2.ts +45 -15
  406. package/src/security/parental-control-store.ts +183 -0
  407. package/src/security/secret-allowlist.ts +4 -3
  408. package/src/security/secret-scanner.ts +5 -5
  409. package/src/security/secure-keys.ts +1 -1
  410. package/src/security/token-manager.ts +3 -2
  411. package/src/services/vercel-deploy.ts +6 -2
  412. package/src/skills/tool-manifest.ts +3 -3
  413. package/src/skills/vellum-catalog-remote.ts +75 -16
  414. package/src/slack/slack-webhook.ts +2 -1
  415. package/src/swarm/orchestrator.ts +92 -1
  416. package/src/swarm/router-planner.ts +6 -9
  417. package/src/swarm/worker-prompts.ts +9 -12
  418. package/src/tasks/task-compiler.ts +19 -28
  419. package/src/tasks/task-runner.ts +1 -1
  420. package/src/tools/assets/search.ts +15 -14
  421. package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
  422. package/src/tools/browser/auto-navigate.ts +1 -0
  423. package/src/tools/browser/browser-execution.ts +13 -1
  424. package/src/tools/browser/browser-manager.ts +119 -4
  425. package/src/tools/browser/network-recorder.ts +5 -0
  426. package/src/tools/credentials/broker.ts +11 -2
  427. package/src/tools/credentials/metadata-store.ts +18 -14
  428. package/src/tools/credentials/post-connect-hooks.ts +61 -0
  429. package/src/tools/credentials/vault.ts +49 -23
  430. package/src/tools/executor.ts +80 -18
  431. package/src/tools/host-terminal/cli-discover.ts +1 -1
  432. package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
  433. package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
  434. package/src/tools/network/script-proxy/server.ts +1 -1
  435. package/src/tools/network/script-proxy/session-manager.ts +6 -5
  436. package/src/tools/network/web-fetch.ts +18 -2
  437. package/src/tools/network/web-search.ts +7 -3
  438. package/src/tools/reminder/reminder-store.ts +14 -15
  439. package/src/tools/schedule/create.ts +1 -0
  440. package/src/tools/schedule/list.ts +2 -1
  441. package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
  442. package/src/tools/skills/skill-script-runner.ts +24 -9
  443. package/src/tools/skills/skill-tool-factory.ts +1 -0
  444. package/src/tools/tasks/work-item-enqueue.ts +2 -2
  445. package/src/tools/terminal/evaluate-typescript.ts +21 -12
  446. package/src/tools/terminal/parser.ts +50 -0
  447. package/src/tools/watcher/delete.ts +6 -0
  448. package/src/tools/weather/service.ts +1 -1
  449. package/src/twitter/client.ts +190 -24
  450. package/src/twitter/session.ts +4 -3
  451. package/src/util/clipboard.ts +1 -1
  452. package/src/util/errors.ts +65 -8
  453. package/src/util/fs.ts +40 -0
  454. package/src/util/json.ts +10 -0
  455. package/src/util/log-redact.ts +189 -0
  456. package/src/util/logger.ts +25 -18
  457. package/src/util/object.ts +3 -0
  458. package/src/util/platform.ts +72 -365
  459. package/src/util/pricing.ts +1 -1
  460. package/src/util/promise-guard.ts +1 -1
  461. package/src/util/retry.ts +19 -0
  462. package/src/util/row-mapper.ts +79 -0
  463. package/src/util/silently.ts +21 -0
  464. package/src/watcher/engine.ts +5 -1
  465. package/src/watcher/provider-types.ts +20 -0
  466. package/src/watcher/providers/github.ts +156 -0
  467. package/src/watcher/providers/gmail.ts +1 -0
  468. package/src/watcher/providers/google-calendar.ts +1 -0
  469. package/src/watcher/providers/linear.ts +460 -0
  470. package/src/watcher/providers/slack.ts +1 -0
  471. package/src/work-items/work-item-runner.ts +1 -1
  472. package/src/workspace/git-service.ts +1 -1
  473. package/src/workspace/provider-commit-message-generator.ts +51 -22
  474. package/src/__tests__/call-bridge.test.ts +0 -517
  475. package/src/__tests__/session-process-bridge.test.ts +0 -244
  476. package/src/calls/call-bridge.ts +0 -168
  477. package/src/config/bundled-skills/media-processing/services/capability-registry.ts +0 -137
  478. package/src/config/bundled-skills/media-processing/services/event-detection-service.ts +0 -280
  479. package/src/config/bundled-skills/media-processing/services/feedback-aggregation.ts +0 -144
  480. package/src/config/bundled-skills/media-processing/services/feedback-store.ts +0 -136
  481. package/src/config/bundled-skills/media-processing/services/retrieval-service.ts +0 -95
  482. package/src/config/bundled-skills/media-processing/services/timeline-service.ts +0 -267
  483. package/src/config/bundled-skills/media-processing/tools/detect-events.ts +0 -110
  484. package/src/config/bundled-skills/media-processing/tools/recalibrate.ts +0 -235
  485. package/src/config/bundled-skills/media-processing/tools/select-tracking-profile.ts +0 -142
  486. package/src/config/bundled-skills/media-processing/tools/submit-feedback.ts +0 -150
  487. package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
@@ -6,8 +6,9 @@
6
6
  * streams text tokens back through the RelayConnection for real-time TTS.
7
7
  */
8
8
 
9
- import Anthropic from '@anthropic-ai/sdk';
10
9
  import { getConfig } from '../config/loader.js';
10
+ import { resolveConfiguredProvider } from '../providers/provider-send-message.js';
11
+ import type { ProviderEvent } from '../providers/types.js';
11
12
  import { resolveUserReference } from '../config/user-reference.js';
12
13
  import { getLogger } from '../util/logger.js';
13
14
  import {
@@ -21,23 +22,38 @@ import { getMaxCallDurationMs, getUserConsultationTimeoutMs, SILENCE_TIMEOUT_MS
21
22
  import type { RelayConnection } from './relay-server.js';
22
23
  import { registerCallOrchestrator, unregisterCallOrchestrator, fireCallQuestionNotifier, fireCallCompletionNotifier, fireCallTranscriptNotifier } from './call-state.js';
23
24
  import type { PromptSpeakerContext } from './speaker-identification.js';
25
+ import { addPointerMessage, formatDuration } from './call-pointer-messages.js';
26
+ import { persistCallCompletionMessage } from './call-conversation-messages.js';
27
+ import * as conversationStore from '../memory/conversation-store.js';
28
+ import { dispatchGuardianQuestion } from './guardian-dispatch.js';
29
+ import type { ServerMessage } from '../daemon/ipc-contract.js';
30
+ import {
31
+ buildGuardianContextBlock,
32
+ type GuardianRuntimeContext,
33
+ } from '../daemon/session-runtime-assembly.js';
24
34
 
25
35
  const log = getLogger('call-orchestrator');
26
36
 
27
37
  type OrchestratorState = 'idle' | 'processing' | 'waiting_on_user' | 'speaking';
28
38
 
29
- const ASK_USER_CAPTURE_REGEX = /\[ASK_USER:\s*(.+?)\]/;
30
- const ASK_USER_MARKER_REGEX = /\[ASK_USER:\s*.+?\]/g;
39
+ const ASK_GUARDIAN_CAPTURE_REGEX = /\[ASK_GUARDIAN:\s*(.+?)\]/;
40
+ const ASK_GUARDIAN_MARKER_REGEX = /\[ASK_GUARDIAN:\s*.+?\]/g;
31
41
  const USER_ANSWERED_MARKER_REGEX = /\[USER_ANSWERED:\s*.+?\]/g;
32
42
  const USER_INSTRUCTION_MARKER_REGEX = /\[USER_INSTRUCTION:\s*.+?\]/g;
43
+ const CALL_OPENING_MARKER_REGEX = /\[CALL_OPENING\]/g;
44
+ const CALL_OPENING_ACK_MARKER_REGEX = /\[CALL_OPENING_ACK\]/g;
33
45
  const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
46
+ const CALL_OPENING_MARKER = '[CALL_OPENING]';
47
+ const CALL_OPENING_ACK_MARKER = '[CALL_OPENING_ACK]';
34
48
  const END_CALL_MARKER = '[END_CALL]';
35
49
 
36
50
  function stripInternalSpeechMarkers(text: string): string {
37
51
  return text
38
- .replace(ASK_USER_MARKER_REGEX, '')
52
+ .replace(ASK_GUARDIAN_MARKER_REGEX, '')
39
53
  .replace(USER_ANSWERED_MARKER_REGEX, '')
40
54
  .replace(USER_INSTRUCTION_MARKER_REGEX, '')
55
+ .replace(CALL_OPENING_MARKER_REGEX, '')
56
+ .replace(CALL_OPENING_ACK_MARKER_REGEX, '')
41
57
  .replace(END_CALL_MARKER_REGEX, '');
42
58
  }
43
59
 
@@ -47,22 +63,46 @@ export class CallOrchestrator {
47
63
  private state: OrchestratorState = 'idle';
48
64
  private conversationHistory: Array<{ role: 'user' | 'assistant'; content: string }> = [];
49
65
  private abortController: AbortController = new AbortController();
50
- private callStartTime: number = Date.now();
51
66
  private silenceTimer: ReturnType<typeof setTimeout> | null = null;
52
67
  private durationTimer: ReturnType<typeof setTimeout> | null = null;
53
68
  private durationWarningTimer: ReturnType<typeof setTimeout> | null = null;
54
69
  private consultationTimer: ReturnType<typeof setTimeout> | null = null;
55
70
  private durationEndTimer: ReturnType<typeof setTimeout> | null = null;
56
71
  private task: string | null;
72
+ /** True when the call session was created via the inbound path (no outbound task). */
73
+ private isInbound: boolean;
57
74
  /** Instructions queued while an LLM turn is in-flight or during waiting_on_user */
58
75
  private pendingInstructions: string[] = [];
76
+ /** Ensures the call opener is triggered at most once per call. */
77
+ private initialGreetingStarted = false;
78
+ /** Marks that the next caller turn should be treated as an opening acknowledgment. */
79
+ private awaitingOpeningAck = false;
59
80
  /** Monotonic run id used to suppress stale turn side effects after interruption. */
60
81
  private llmRunVersion = 0;
61
-
62
- constructor(callSessionId: string, relay: RelayConnection, task: string | null) {
82
+ /** Optional broadcast function for emitting IPC events to connected clients. */
83
+ private broadcast?: (msg: ServerMessage) => void;
84
+ /** Assistant identity for scoping guardian bindings. */
85
+ private assistantId: string;
86
+ /** Guardian trust context for the current caller, when available. */
87
+ private guardianContext: GuardianRuntimeContext | null;
88
+
89
+ constructor(
90
+ callSessionId: string,
91
+ relay: RelayConnection,
92
+ task: string | null,
93
+ opts?: {
94
+ broadcast?: (msg: ServerMessage) => void;
95
+ assistantId?: string;
96
+ guardianContext?: GuardianRuntimeContext;
97
+ },
98
+ ) {
63
99
  this.callSessionId = callSessionId;
64
100
  this.relay = relay;
65
101
  this.task = task;
102
+ this.isInbound = !task;
103
+ this.broadcast = opts?.broadcast;
104
+ this.assistantId = opts?.assistantId ?? 'self';
105
+ this.guardianContext = opts?.guardianContext ?? null;
66
106
  this.startDurationTimer();
67
107
  this.resetSilenceTimer();
68
108
  registerCallOrchestrator(callSessionId, this);
@@ -75,6 +115,30 @@ export class CallOrchestrator {
75
115
  return this.state;
76
116
  }
77
117
 
118
+ /**
119
+ * Update guardian trust context for subsequent LLM turns.
120
+ */
121
+ setGuardianContext(ctx: GuardianRuntimeContext | null): void {
122
+ this.guardianContext = ctx;
123
+ }
124
+
125
+ /**
126
+ * Kick off the first outbound call utterance from the assistant.
127
+ */
128
+ async startInitialGreeting(): Promise<void> {
129
+ if (this.initialGreetingStarted) return;
130
+ if (this.state !== 'idle') return;
131
+
132
+ this.initialGreetingStarted = true;
133
+ this.resetSilenceTimer();
134
+ this.conversationHistory.push({ role: 'user', content: CALL_OPENING_MARKER });
135
+ await this.runLlm();
136
+ const lastMessage = this.conversationHistory[this.conversationHistory.length - 1];
137
+ if (lastMessage?.role === 'assistant') {
138
+ this.awaitingOpeningAck = true;
139
+ }
140
+ }
141
+
78
142
  /**
79
143
  * Handle a final caller utterance from the ConversationRelay.
80
144
  */
@@ -86,9 +150,42 @@ export class CallOrchestrator {
86
150
  this.abortController = new AbortController();
87
151
  }
88
152
 
153
+ // Strip the one-shot [CALL_OPENING] marker from conversation history
154
+ // so it doesn't leak into subsequent LLM requests after barge-in.
155
+ // This runs unconditionally because the standard Twilio barge-in path
156
+ // calls handleInterrupt() first (setting state to 'idle') before
157
+ // handleCallerUtterance — so interruptedInFlight would be false even
158
+ // though an interrupt just occurred.
159
+ // Without this, the consecutive-user merge path below would append
160
+ // the caller's transcript to the synthetic "[CALL_OPENING]" message,
161
+ // causing the model to re-run opener behavior instead of responding
162
+ // directly to the caller.
163
+ // If the marker-only seed message becomes empty, remove it entirely:
164
+ // Anthropic rejects any user turn with empty content.
165
+ for (let i = 0; i < this.conversationHistory.length; i++) {
166
+ const entry = this.conversationHistory[i];
167
+ if (!entry.content.includes(CALL_OPENING_MARKER)) continue;
168
+ const stripped = entry.content.replace(CALL_OPENING_MARKER_REGEX, '').trim();
169
+ if (stripped.length === 0) {
170
+ this.conversationHistory.splice(i, 1);
171
+ i--;
172
+ } else {
173
+ entry.content = stripped;
174
+ }
175
+ }
176
+
89
177
  this.state = 'processing';
90
178
  this.resetSilenceTimer();
91
179
  const callerContent = this.formatCallerUtterance(transcript, speaker);
180
+ const shouldMarkOpeningAck = this.awaitingOpeningAck;
181
+ if (shouldMarkOpeningAck) {
182
+ this.awaitingOpeningAck = false;
183
+ }
184
+ const callerTurnContent = shouldMarkOpeningAck
185
+ ? callerContent.length > 0
186
+ ? `${CALL_OPENING_ACK_MARKER}\n${callerContent}`
187
+ : CALL_OPENING_ACK_MARKER
188
+ : callerContent;
92
189
 
93
190
  // Preserve strict role alternation for Anthropic. If the last message
94
191
  // is already user-role (e.g. interrupted run never appended assistant,
@@ -96,11 +193,14 @@ export class CallOrchestrator {
96
193
  // this utterance into that same user turn.
97
194
  const lastMessage = this.conversationHistory[this.conversationHistory.length - 1];
98
195
  if (lastMessage?.role === 'user') {
99
- lastMessage.content = `${lastMessage.content}\n${callerContent}`;
196
+ const existingContent = lastMessage.content.trim();
197
+ lastMessage.content = existingContent.length > 0
198
+ ? `${lastMessage.content}\n${callerTurnContent}`
199
+ : callerTurnContent;
100
200
  } else {
101
201
  this.conversationHistory.push({
102
202
  role: 'user',
103
- content: callerContent,
203
+ content: callerTurnContent,
104
204
  });
105
205
  }
106
206
 
@@ -221,30 +321,83 @@ export class CallOrchestrator {
221
321
 
222
322
  // ── Private ──────────────────────────────────────────────────────
223
323
 
324
+ private buildGuardianPromptSection(): string[] {
325
+ if (!this.guardianContext) return [];
326
+ return [
327
+ '',
328
+ 'GUARDIAN ACTOR CONTEXT (authoritative):',
329
+ buildGuardianContextBlock(this.guardianContext),
330
+ '- Treat `actor_role` as source-of-truth for whether this caller is the verified guardian.',
331
+ '- If `actor_role` is `guardian`, the current caller is verified for this assistant on voice.',
332
+ '- If `actor_role` is `non-guardian` or `unverified_channel`, do not imply the caller is verified.',
333
+ ];
334
+ }
335
+
224
336
  private buildSystemPrompt(): string {
225
337
  const config = getConfig();
226
338
  const disclosureRule = config.calls.disclosure.enabled
227
339
  ? `1. ${config.calls.disclosure.text}`
228
340
  : '1. Begin the conversation naturally.';
229
341
 
342
+ if (this.isInbound) {
343
+ return this.buildInboundSystemPrompt(disclosureRule);
344
+ }
345
+
230
346
  return [
231
347
  `You are on a live phone call on behalf of ${resolveUserReference()}.`,
232
348
  this.task ? `Task: ${this.task}` : '',
233
349
  '',
234
350
  'You are speaking directly to the person who answered the phone.',
235
351
  'Respond naturally and conversationally — speak as you would in a real phone conversation.',
352
+ ...this.buildGuardianPromptSection(),
236
353
  '',
237
354
  'IMPORTANT RULES:',
238
355
  '0. When introducing yourself, refer to yourself as an assistant. Avoid the phrase "AI assistant" unless directly asked.',
239
356
  disclosureRule,
240
357
  '2. Be concise — phone conversations should be brief and natural.',
241
- '3. If the callee asks something you don\'t know, include [ASK_USER: your question here] in your response along with a hold message like "Let me check on that for you."',
358
+ '3. If the callee asks something you don\'t know, include [ASK_GUARDIAN: your question here] in your response along with a hold message like "Let me check on that for you."',
242
359
  '4. If the callee provides information preceded by [USER_ANSWERED: ...], use that answer naturally in the conversation.',
243
360
  '5. If you see [USER_INSTRUCTION: ...], treat it as a high-priority steering directive from your user. Follow the instruction immediately, adjusting your approach or response accordingly.',
244
361
  '6. When the call\'s purpose is fulfilled, include [END_CALL] in your response along with a polite goodbye.',
245
362
  '7. Do not make up information — ask the user if unsure.',
246
363
  '8. Keep responses short — 1-3 sentences is ideal for phone conversation.',
247
364
  '9. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
365
+ '10. If the latest user turn is [CALL_OPENING], generate a natural, context-specific opener: briefly introduce yourself once as an assistant, state why you are calling using the Task context, and ask a short permission/check-in question. Vary the wording; do not use a fixed template.',
366
+ '11. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.',
367
+ '12. Do not repeat your introduction within the same call unless the callee explicitly asks who you are.',
368
+ ]
369
+ .filter(Boolean)
370
+ .join('\n');
371
+ }
372
+
373
+ /**
374
+ * Build a system prompt tailored for inbound calls where the caller
375
+ * reached out to us. The assistant greets naturally and helps the
376
+ * caller with whatever they need, rather than delivering an outbound
377
+ * task message.
378
+ */
379
+ private buildInboundSystemPrompt(disclosureRule: string): string {
380
+ return [
381
+ `You are on a live phone call, answering an incoming call on behalf of ${resolveUserReference()}.`,
382
+ '',
383
+ 'The caller dialed in to reach you. You do not have a specific task — your role is to greet them warmly, find out what they need, and assist them.',
384
+ 'Respond naturally and conversationally — speak as you would in a real phone conversation.',
385
+ ...this.buildGuardianPromptSection(),
386
+ '',
387
+ 'IMPORTANT RULES:',
388
+ '0. When introducing yourself, refer to yourself as an assistant. Avoid the phrase "AI assistant" unless directly asked.',
389
+ disclosureRule,
390
+ '2. Be concise — phone conversations should be brief and natural.',
391
+ '3. If the caller asks something you don\'t know or need to verify, include [ASK_GUARDIAN: your question here] in your response along with a hold message like "Let me check on that for you."',
392
+ '4. If information is provided preceded by [USER_ANSWERED: ...], use that answer naturally in the conversation.',
393
+ '5. If you see [USER_INSTRUCTION: ...], treat it as a high-priority steering directive from your user. Follow the instruction immediately, adjusting your approach or response accordingly.',
394
+ '6. When the caller indicates they are done or the conversation reaches a natural conclusion, include [END_CALL] in your response along with a polite goodbye.',
395
+ '7. Do not make up information — ask the user if unsure.',
396
+ '8. Keep responses short — 1-3 sentences is ideal for phone conversation.',
397
+ '9. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
398
+ '10. If the latest user turn is [CALL_OPENING], greet the caller warmly and ask how you can help. For example: "Hello, this is [name]\'s assistant. How can I help you today?" Vary the wording; do not use a fixed template.',
399
+ '11. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.',
400
+ '12. Do not repeat your introduction within the same call unless the caller explicitly asks who you are.',
248
401
  ]
249
402
  .filter(Boolean)
250
403
  .join('\n');
@@ -254,7 +407,7 @@ export class CallOrchestrator {
254
407
  if (!speaker) return transcript;
255
408
  const safeId = speaker.speakerId.replaceAll('"', '\'');
256
409
  const safeLabel = speaker.speakerLabel.replaceAll('"', '\'');
257
- const confidencePart = speaker.speakerConfidence !== null ? ` confidence="${speaker.speakerConfidence.toFixed(2)}"` : '';
410
+ const confidencePart = speaker.speakerConfidence != null ? ` confidence="${speaker.speakerConfidence.toFixed(2)}"` : '';
258
411
  return `[SPEAKER id="${safeId}" label="${safeLabel}" source="${speaker.source}"${confidencePart}] ${transcript}`;
259
412
  }
260
413
 
@@ -263,37 +416,31 @@ export class CallOrchestrator {
263
416
  * the response back through the relay.
264
417
  */
265
418
  private async runLlm(): Promise<void> {
266
- const apiKey = getConfig().apiKeys.anthropic ?? process.env.ANTHROPIC_API_KEY;
267
- if (!apiKey) {
268
- log.error({ callSessionId: this.callSessionId }, 'No Anthropic API key available');
419
+ const config = getConfig();
420
+ const resolved = resolveConfiguredProvider();
421
+ if (!resolved) {
422
+ log.error({ callSessionId: this.callSessionId }, 'No provider available');
269
423
  this.relay.sendTextToken('I\'m sorry, I\'m having a technical issue. Please try again later.', true);
270
424
  this.state = 'idle';
271
425
  return;
272
426
  }
427
+ const { provider } = resolved;
273
428
 
274
- const client = new Anthropic({ apiKey });
275
429
  const runVersion = ++this.llmRunVersion;
276
430
  const runSignal = this.abortController.signal;
277
431
 
278
432
  try {
279
433
  this.state = 'speaking';
280
434
 
281
- const callModel = getConfig().calls.model?.trim() || 'claude-sonnet-4-20250514';
282
-
283
- const stream = client.messages.stream(
284
- {
285
- model: callModel,
286
- max_tokens: 512,
287
- system: this.buildSystemPrompt(),
288
- messages: this.conversationHistory.map((m) => ({
289
- role: m.role,
290
- content: m.content,
291
- })),
292
- },
293
- { signal: runSignal },
294
- );
435
+ // Only override the model when the user has explicitly configured one
436
+ // AND the selected provider matches the configured provider. Forwarding
437
+ // a provider-specific model to a fallback provider would cause
438
+ // cross-provider 4xx errors (e.g., sending "gpt-5.2" to Anthropic).
439
+ const callModel = !resolved.usedFallbackPrimary
440
+ ? (config.calls.model?.trim() || undefined)
441
+ : undefined;
295
442
 
296
- // Buffer incoming tokens so we can strip control markers ([ASK_USER:...], [END_CALL])
443
+ // Buffer incoming tokens so we can strip control markers ([ASK_GUARDIAN:...], [END_CALL])
297
444
  // before they reach TTS. We hold text whenever an unmatched '[' appears, since it
298
445
  // could be the start of a control marker.
299
446
  let ttsBuffer = '';
@@ -320,18 +467,24 @@ export class CallOrchestrator {
320
467
  // The check must be bidirectional:
321
468
  // - When the buffer is shorter than the prefix (e.g. "[ASK"), the
322
469
  // buffer is a prefix of the control tag → hold it.
323
- // - When the buffer is longer than the prefix (e.g. "[ASK_USER: what"),
470
+ // - When the buffer is longer than the prefix (e.g. "[ASK_GUARDIAN: what"),
324
471
  // the buffer starts with the control tag prefix → hold it (the
325
472
  // variable-length payload hasn't been closed yet).
326
473
  const afterBracket = ttsBuffer;
327
474
  const couldBeControl =
328
- '[ASK_USER:'.startsWith(afterBracket) ||
475
+ '[ASK_GUARDIAN:'.startsWith(afterBracket) ||
329
476
  '[USER_ANSWERED:'.startsWith(afterBracket) ||
330
477
  '[USER_INSTRUCTION:'.startsWith(afterBracket) ||
478
+ '[CALL_OPENING]'.startsWith(afterBracket) ||
479
+ '[CALL_OPENING_ACK]'.startsWith(afterBracket) ||
331
480
  '[END_CALL]'.startsWith(afterBracket) ||
332
- afterBracket.startsWith('[ASK_USER:') ||
481
+ afterBracket.startsWith('[ASK_GUARDIAN:') ||
333
482
  afterBracket.startsWith('[USER_ANSWERED:') ||
334
483
  afterBracket.startsWith('[USER_INSTRUCTION:') ||
484
+ afterBracket === '[CALL_OPENING' ||
485
+ afterBracket.startsWith('[CALL_OPENING]') ||
486
+ afterBracket === '[CALL_OPENING_ACK' ||
487
+ afterBracket.startsWith('[CALL_OPENING_ACK]') ||
335
488
  afterBracket === '[END_CALL' ||
336
489
  afterBracket.startsWith('[END_CALL]');
337
490
 
@@ -351,17 +504,29 @@ export class CallOrchestrator {
351
504
  }
352
505
  };
353
506
 
354
- stream.on('text', (text) => {
355
- if (!this.isCurrentRun(runVersion)) return;
356
- ttsBuffer += text;
357
-
358
- // Remove complete control markers before text reaches TTS.
359
- ttsBuffer = stripInternalSpeechMarkers(ttsBuffer);
360
-
361
- flushSafeText(false);
362
- });
363
-
364
- const finalMessage = await stream.finalMessage();
507
+ const response = await provider.sendMessage(
508
+ this.conversationHistory.map((m) => ({
509
+ role: m.role as 'user' | 'assistant',
510
+ content: [{ type: 'text' as const, text: m.content }],
511
+ })),
512
+ [], // no tools
513
+ this.buildSystemPrompt(),
514
+ {
515
+ config: {
516
+ ...(callModel ? { model: callModel } : {}),
517
+ max_tokens: 512,
518
+ },
519
+ onEvent: (event: ProviderEvent) => {
520
+ if (!this.isCurrentRun(runVersion)) return;
521
+ if (event.type === 'text_delta') {
522
+ ttsBuffer += event.text;
523
+ ttsBuffer = stripInternalSpeechMarkers(ttsBuffer);
524
+ flushSafeText(false);
525
+ }
526
+ },
527
+ signal: runSignal,
528
+ },
529
+ );
365
530
  if (!this.isCurrentRun(runVersion)) return;
366
531
 
367
532
  // Final sweep: strip any remaining control markers from the buffer
@@ -373,11 +538,10 @@ export class CallOrchestrator {
373
538
  // Signal end of this turn's speech
374
539
  this.relay.sendTextToken('', true);
375
540
 
376
- const responseText =
377
- finalMessage.content
378
- .filter((b): b is Anthropic.TextBlock => b.type === 'text')
379
- .map((b) => b.text)
380
- .join('') || '';
541
+ const responseText = response.content
542
+ .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
543
+ .map((b) => b.text)
544
+ .join('') || '';
381
545
 
382
546
  // Record the assistant response
383
547
  this.conversationHistory.push({ role: 'assistant', content: responseText });
@@ -386,15 +550,23 @@ export class CallOrchestrator {
386
550
  if (spokenText.length > 0) {
387
551
  const session = getCallSession(this.callSessionId);
388
552
  if (session) {
553
+ // Persist assistant transcript to the voice conversation so it
554
+ // survives even when no live daemon Session is listening.
555
+ conversationStore.addMessage(
556
+ session.conversationId,
557
+ 'assistant',
558
+ JSON.stringify([{ type: 'text', text: spokenText }]),
559
+ { userMessageChannel: 'voice', assistantMessageChannel: 'voice' },
560
+ );
389
561
  fireCallTranscriptNotifier(session.conversationId, this.callSessionId, 'assistant', spokenText);
390
562
  }
391
563
  }
392
564
 
393
- // Check for ASK_USER pattern
394
- const askMatch = responseText.match(ASK_USER_CAPTURE_REGEX);
565
+ // Check for ASK_GUARDIAN pattern
566
+ const askMatch = responseText.match(ASK_GUARDIAN_CAPTURE_REGEX);
395
567
  if (askMatch) {
396
568
  const questionText = askMatch[1];
397
- createPendingQuestion(this.callSessionId, questionText);
569
+ const pendingQuestion = createPendingQuestion(this.callSessionId, questionText);
398
570
  this.state = 'waiting_on_user';
399
571
  updateCallSession(this.callSessionId, { status: 'waiting_on_user' });
400
572
  recordCallEvent(this.callSessionId, 'user_question_asked', { question: questionText });
@@ -403,6 +575,15 @@ export class CallOrchestrator {
403
575
  const session = getCallSession(this.callSessionId);
404
576
  if (session) {
405
577
  fireCallQuestionNotifier(session.conversationId, this.callSessionId, questionText);
578
+
579
+ // Dispatch guardian action request to all configured channels
580
+ void dispatchGuardianQuestion({
581
+ callSessionId: this.callSessionId,
582
+ conversationId: session.conversationId,
583
+ assistantId: this.assistantId,
584
+ pendingQuestion,
585
+ broadcast: this.broadcast,
586
+ });
406
587
  }
407
588
 
408
589
  // Set a consultation timeout
@@ -433,11 +614,19 @@ export class CallOrchestrator {
433
614
  updateCallSession(this.callSessionId, { status: 'completed', endedAt: Date.now() });
434
615
  recordCallEvent(this.callSessionId, 'call_ended', { reason: 'completed' });
435
616
 
436
- // Notify the conversation when this is the first transition
437
- // into a terminal call state.
617
+ // Notify the voice conversation
438
618
  if (shouldNotifyCompletion && currentSession) {
619
+ persistCallCompletionMessage(currentSession.conversationId, this.callSessionId);
439
620
  fireCallCompletionNotifier(currentSession.conversationId, this.callSessionId);
440
621
  }
622
+
623
+ // Post a pointer message in the initiating conversation
624
+ if (currentSession?.initiatedFromConversationId) {
625
+ const durationMs = currentSession.startedAt ? Date.now() - currentSession.startedAt : 0;
626
+ addPointerMessage(currentSession.initiatedFromConversationId, 'completed', currentSession.toNumber, {
627
+ duration: durationMs > 0 ? formatDuration(durationMs) : undefined,
628
+ });
629
+ }
441
630
  this.state = 'idle';
442
631
  return;
443
632
  }
@@ -545,8 +734,17 @@ export class CallOrchestrator {
545
734
  updateCallSession(this.callSessionId, { status: 'completed', endedAt: Date.now() });
546
735
  recordCallEvent(this.callSessionId, 'call_ended', { reason: 'max_duration' });
547
736
  if (shouldNotifyCompletion && currentSession) {
737
+ persistCallCompletionMessage(currentSession.conversationId, this.callSessionId);
548
738
  fireCallCompletionNotifier(currentSession.conversationId, this.callSessionId);
549
739
  }
740
+
741
+ // Post a pointer message in the initiating conversation
742
+ if (currentSession?.initiatedFromConversationId) {
743
+ const durationMs = currentSession.startedAt ? Date.now() - currentSession.startedAt : 0;
744
+ addPointerMessage(currentSession.initiatedFromConversationId, 'completed', currentSession.toNumber, {
745
+ duration: durationMs > 0 ? formatDuration(durationMs) : undefined,
746
+ });
747
+ }
550
748
  }, 3000);
551
749
  }, maxDurationMs);
552
750
  }
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Concise pointer/status messages posted to the initiating conversation
3
+ * so the user sees call lifecycle events without the full transcript
4
+ * (which lives in the dedicated voice conversation).
5
+ */
6
+
7
+ import * as conversationStore from '../memory/conversation-store.js';
8
+
9
+ export type PointerEvent = 'started' | 'completed' | 'failed';
10
+
11
+ export function addPointerMessage(
12
+ conversationId: string,
13
+ event: PointerEvent,
14
+ phoneNumber: string,
15
+ extra?: { duration?: string; reason?: string; verificationCode?: string },
16
+ ): void {
17
+ let text: string;
18
+ switch (event) {
19
+ case 'started':
20
+ text = extra?.verificationCode
21
+ ? `\u{1F4DE} Call to ${phoneNumber} started. Verification code: ${extra.verificationCode}`
22
+ : `\u{1F4DE} Call to ${phoneNumber} started. See voice thread for details.`;
23
+ break;
24
+ case 'completed':
25
+ text = extra?.duration
26
+ ? `\u{1F4DE} Call to ${phoneNumber} completed (${extra.duration}).`
27
+ : `\u{1F4DE} Call to ${phoneNumber} completed.`;
28
+ break;
29
+ case 'failed':
30
+ text = extra?.reason
31
+ ? `\u{1F4DE} Call to ${phoneNumber} failed: ${extra.reason}.`
32
+ : `\u{1F4DE} Call to ${phoneNumber} failed.`;
33
+ break;
34
+ }
35
+
36
+ conversationStore.addMessage(
37
+ conversationId,
38
+ 'assistant',
39
+ JSON.stringify([{ type: 'text', text }]),
40
+ { userMessageChannel: 'voice', assistantMessageChannel: 'voice' },
41
+ );
42
+ }
43
+
44
+ /**
45
+ * Format a duration in milliseconds into a human-friendly string.
46
+ */
47
+ export function formatDuration(ms: number): string {
48
+ const totalSeconds = Math.round(ms / 1000);
49
+ if (totalSeconds < 60) return `${totalSeconds}s`;
50
+ const minutes = Math.floor(totalSeconds / 60);
51
+ const seconds = totalSeconds % 60;
52
+ return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
53
+ }
@@ -8,9 +8,10 @@
8
8
  */
9
9
 
10
10
  import { getLogger } from '../util/logger.js';
11
+ import { isTerminalState } from './call-state-machine.js';
11
12
  import { listRecoverableCalls, updateCallSession, expirePendingQuestions } from './call-store.js';
12
- import type { VoiceProvider } from './voice-provider.js';
13
13
  import type { CallStatus } from './types.js';
14
+ import type { VoiceProvider } from './voice-provider.js';
14
15
 
15
16
  type Logger = ReturnType<typeof getLogger>;
16
17
 
@@ -57,12 +58,6 @@ function mapProviderStatus(providerStatus: string): CallStatus | null {
57
58
  }
58
59
  }
59
60
 
60
- /**
61
- * Check whether a CallStatus is terminal (no further transitions allowed).
62
- */
63
- function isTerminal(status: CallStatus): boolean {
64
- return status === 'completed' || status === 'failed' || status === 'cancelled';
65
- }
66
61
 
67
62
  /**
68
63
  * Reconcile all non-terminal call sessions at daemon startup.
@@ -159,7 +154,7 @@ export async function reconcileCallsOnStartup(
159
154
  continue;
160
155
  }
161
156
 
162
- if (isTerminal(mappedStatus)) {
157
+ if (isTerminalState(mappedStatus)) {
163
158
  // Provider says the call has ended
164
159
  log.info(
165
160
  { callSessionId: session.id, providerStatus, mappedStatus },