@vellumai/assistant 0.12.1 → 0.12.2-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (548) hide show
  1. package/AGENTS.md +4 -0
  2. package/ARCHITECTURE.md +20 -21
  3. package/docs/architecture/turn-actor.md +15 -8
  4. package/docs/guardian-request-flow.md +16 -0
  5. package/openapi.yaml +327 -52
  6. package/package.json +3 -1
  7. package/scripts/smoke-desktop-window-theme.ts +186 -0
  8. package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
  9. package/src/__tests__/anthropic-provider.test.ts +0 -63
  10. package/src/__tests__/app-compiler.test.ts +6 -0
  11. package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
  12. package/src/__tests__/app-dir-path-guard.test.ts +1 -3
  13. package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
  14. package/src/__tests__/assistant-event-hub.test.ts +44 -0
  15. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  16. package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
  17. package/src/__tests__/computer-use-tools.test.ts +19 -7
  18. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  19. package/src/__tests__/config-schema.test.ts +28 -29
  20. package/src/__tests__/config-watcher.test.ts +23 -8
  21. package/src/__tests__/connection-model-compat.test.ts +2 -2
  22. package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
  23. package/src/__tests__/conversation-pairing.test.ts +26 -0
  24. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
  25. package/src/__tests__/conversation-queue.test.ts +48 -207
  26. package/src/__tests__/conversation-retry-discard.test.ts +54 -1
  27. package/src/__tests__/conversation-skill-tools.test.ts +27 -0
  28. package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
  29. package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
  30. package/src/__tests__/conversation-title-service.test.ts +14 -10
  31. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
  32. package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
  33. package/src/__tests__/credential-record-retire.test.ts +171 -1
  34. package/src/__tests__/credential-routes.test.ts +212 -3
  35. package/src/__tests__/credential-security-invariants.test.ts +4 -2
  36. package/src/__tests__/cu-unified-flow.test.ts +8 -2
  37. package/src/__tests__/document-sync-tags.test.ts +25 -0
  38. package/src/__tests__/drain-kick-guard.test.ts +0 -2
  39. package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
  40. package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
  41. package/src/__tests__/git-porcelain.test.ts +47 -0
  42. package/src/__tests__/heartbeat-service.test.ts +56 -40
  43. package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
  44. package/src/__tests__/host-bash-proxy.test.ts +26 -45
  45. package/src/__tests__/host-browser-proxy.test.ts +6 -17
  46. package/src/__tests__/host-cu-proxy.test.ts +539 -0
  47. package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
  48. package/src/__tests__/host-file-proxy.test.ts +11 -23
  49. package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
  50. package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
  51. package/src/__tests__/llm-catalog-parity.test.ts +25 -0
  52. package/src/__tests__/mcp-auth-routes.test.ts +6 -9
  53. package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
  54. package/src/__tests__/mcp-health-check.test.ts +8 -11
  55. package/src/__tests__/mcp-list-plugin-servers.test.ts +8 -10
  56. package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
  57. package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
  58. package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
  59. package/src/__tests__/messaging-send-tool.test.ts +355 -548
  60. package/src/__tests__/model-intents.test.ts +3 -0
  61. package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
  62. package/src/__tests__/oauth-commands-routes.test.ts +113 -0
  63. package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
  64. package/src/__tests__/openai-provider.test.ts +1 -0
  65. package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
  66. package/src/__tests__/platform-callback-registration.test.ts +8 -16
  67. package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
  68. package/src/__tests__/plugin-import-boundary-guard.test.ts +14 -0
  69. package/src/__tests__/qdrant-manager.test.ts +132 -0
  70. package/src/__tests__/resolve-app.test.ts +135 -0
  71. package/src/__tests__/scaffold-managed-skill-tool.test.ts +229 -6
  72. package/src/__tests__/script-proxy-certs.test.ts +35 -0
  73. package/src/__tests__/service-account-token.test.ts +247 -0
  74. package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
  75. package/src/__tests__/skill-feature-flags.test.ts +2 -2
  76. package/src/__tests__/skills.test.ts +9 -4
  77. package/src/__tests__/source-fingerprint.test.ts +184 -0
  78. package/src/__tests__/subagent-tool-gate-mode.test.ts +19 -3
  79. package/src/__tests__/subagent-tools.test.ts +3 -2
  80. package/src/__tests__/telephony-tts-guard.test.ts +38 -3
  81. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  82. package/src/__tests__/unicode.test.ts +13 -0
  83. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
  84. package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
  85. package/src/__tests__/workspace-git-service.test.ts +26 -0
  86. package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
  87. package/src/acp/__tests__/agent-process.test.ts +258 -4
  88. package/src/acp/__tests__/auth-required.test.ts +96 -0
  89. package/src/acp/__tests__/client-handler.test.ts +90 -2
  90. package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
  91. package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
  92. package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
  93. package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
  94. package/src/acp/__tests__/session-manager.test.ts +895 -4
  95. package/src/acp/agent-process.test.ts +4 -1
  96. package/src/acp/agent-process.ts +96 -10
  97. package/src/acp/auth-required.ts +50 -1
  98. package/src/acp/auto-install.test.ts +9 -7
  99. package/src/acp/client-handler.ts +16 -3
  100. package/src/acp/model-config.test.ts +166 -0
  101. package/src/acp/model-config.ts +138 -0
  102. package/src/acp/resolve-agent.test.ts +220 -6
  103. package/src/acp/resolve-agent.ts +88 -23
  104. package/src/acp/session-manager.test.ts +196 -3
  105. package/src/acp/session-manager.ts +441 -18
  106. package/src/acp/types.ts +50 -5
  107. package/src/agent/loop.ts +1 -0
  108. package/src/api/events/acp-session-model-update.test.ts +88 -0
  109. package/src/api/events/acp-session-model-update.ts +46 -0
  110. package/src/api/events/app-files-changed.ts +4 -3
  111. package/src/api/events/notification-intent.ts +2 -0
  112. package/src/api/events/user-message-echo.ts +7 -1
  113. package/src/api/index.ts +6 -0
  114. package/src/api/responses/conversation-message.ts +5 -0
  115. package/src/apps/app-store.ts +1 -1
  116. package/src/apps/resolve-app.ts +81 -0
  117. package/src/apps/source-fingerprint.ts +266 -0
  118. package/src/avatar/identity-avatar.ts +7 -2
  119. package/src/bundler/app-compiler.ts +15 -0
  120. package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
  121. package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
  122. package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
  123. package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
  124. package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
  125. package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
  126. package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
  127. package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
  128. package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +4 -4
  129. package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
  130. package/src/calls/__tests__/voice-control-protocol.test.ts +58 -25
  131. package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
  132. package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
  133. package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
  134. package/src/calls/__tests__/voice-session-bridge.test.ts +37 -1
  135. package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
  136. package/src/calls/barge-in-guard.ts +118 -0
  137. package/src/calls/call-constants.ts +24 -4
  138. package/src/calls/call-controller.ts +499 -199
  139. package/src/calls/call-transport.ts +16 -0
  140. package/src/calls/media-stream-output.ts +32 -0
  141. package/src/calls/media-stream-server.ts +82 -30
  142. package/src/calls/media-stream-stt-session.ts +15 -0
  143. package/src/{live-voice → calls}/progress-narration.ts +2 -2
  144. package/src/calls/resolve-call-tts-provider.ts +33 -2
  145. package/src/calls/tts-call-strategy.ts +9 -5
  146. package/src/calls/types.ts +2 -0
  147. package/src/calls/voice-control-protocol.ts +53 -21
  148. package/src/calls/voice-leg-coordinator.ts +189 -0
  149. package/src/calls/voice-progress-cadence.ts +423 -0
  150. package/src/calls/voice-session-bridge.ts +59 -46
  151. package/src/calls/voice-triage-escalate.ts +175 -56
  152. package/src/cli/commands/__tests__/apps.test.ts +245 -0
  153. package/src/cli/commands/__tests__/bash.test.ts +53 -0
  154. package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
  155. package/src/cli/commands/apps.help.ts +67 -13
  156. package/src/cli/commands/apps.ts +183 -0
  157. package/src/cli/commands/bash.help.ts +8 -13
  158. package/src/cli/commands/bash.ts +7 -25
  159. package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
  160. package/src/cli/commands/channels/index.help.ts +70 -2
  161. package/src/cli/commands/channels/index.ts +16 -3
  162. package/src/cli/commands/channels/send.ts +128 -0
  163. package/src/cli/commands/credentials.help.ts +6 -2
  164. package/src/cli/commands/credentials.ts +2 -2
  165. package/src/cli/commands/mcp.help.ts +8 -8
  166. package/src/cli/commands/mcp.ts +1 -1
  167. package/src/cli/commands/roadmap.help.ts +5 -1
  168. package/src/cli/commands/roadmap.ts +1 -1
  169. package/src/cli/lib/bundled-marketplace.json +103 -90
  170. package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
  171. package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
  172. package/src/config/__tests__/platform-identity.test.ts +255 -0
  173. package/src/config/__tests__/platform-rehydration.test.ts +99 -9
  174. package/src/config/acp-agent-write.ts +43 -0
  175. package/src/config/acp-defaults.test.ts +13 -3
  176. package/src/config/acp-defaults.ts +23 -6
  177. package/src/config/acp-schema.test.ts +81 -0
  178. package/src/config/acp-schema.ts +30 -2
  179. package/src/config/bundled-skills/acp/SKILL.md +22 -14
  180. package/src/config/bundled-skills/acp/TOOLS.json +5 -1
  181. package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
  182. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
  183. package/src/config/bundled-skills/computer-use/SKILL.md +20 -2
  184. package/src/config/bundled-skills/computer-use/TOOLS.json +99 -0
  185. package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
  186. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
  187. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
  188. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
  189. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  190. package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
  191. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
  192. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
  193. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
  194. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
  195. package/src/config/bundled-skills/schedule/SKILL.md +3 -4
  196. package/src/config/bundled-skills/screen-annotation/SKILL.md +8 -1
  197. package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
  198. package/src/config/bundled-skills/skill-management/TOOLS.json +6 -2
  199. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
  200. package/src/config/default-profile-catalog.ts +1 -1
  201. package/src/config/env.ts +2 -2
  202. package/src/config/feature-flag-registry.json +24 -12
  203. package/src/config/loader.ts +1 -0
  204. package/src/config/memory-v3-gate.test.ts +31 -0
  205. package/src/config/memory-v3-gate.ts +20 -16
  206. package/src/config/platform-identity.ts +201 -0
  207. package/src/config/platform-rehydration.ts +11 -27
  208. package/src/config/schema.ts +2 -2
  209. package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
  210. package/src/config/schemas/__tests__/voice.test.ts +148 -0
  211. package/src/config/schemas/calls.ts +1 -28
  212. package/src/config/schemas/live-voice.ts +1 -173
  213. package/src/config/schemas/mcp.ts +1 -1
  214. package/src/config/schemas/memory-retrospective.ts +10 -0
  215. package/src/config/schemas/services.ts +8 -0
  216. package/src/config/schemas/skills.ts +0 -14
  217. package/src/config/schemas/voice.ts +150 -0
  218. package/src/credential-execution/process-manager.test.ts +18 -19
  219. package/src/credential-health/credential-health-service.ts +11 -17
  220. package/src/daemon/__tests__/conversation-tool-setup.test.ts +66 -121
  221. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +4 -1
  222. package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
  223. package/src/daemon/app-change-notify.ts +45 -0
  224. package/src/daemon/config-watcher.ts +13 -194
  225. package/src/daemon/conversation-error.ts +3 -2
  226. package/src/daemon/conversation-media-retry.ts +2 -1
  227. package/src/daemon/conversation-process.ts +35 -64
  228. package/src/daemon/conversation-runtime-assembly.ts +2 -1
  229. package/src/daemon/conversation-skill-tools.ts +13 -31
  230. package/src/daemon/conversation-store.ts +1 -1
  231. package/src/daemon/conversation-surfaces.ts +15 -9
  232. package/src/daemon/conversation-tool-setup.ts +37 -43
  233. package/src/daemon/conversation.ts +0 -3
  234. package/src/daemon/dictation-profile-store.ts +2 -1
  235. package/src/daemon/handlers/shared.ts +2 -1
  236. package/src/daemon/handlers/skills.ts +6 -3
  237. package/src/daemon/host-bash-proxy.ts +8 -12
  238. package/src/daemon/host-browser-proxy.ts +15 -1
  239. package/src/daemon/host-cu-proxy.ts +186 -19
  240. package/src/daemon/host-file-proxy.ts +8 -10
  241. package/src/daemon/host-proxy-preactivation.ts +2 -1
  242. package/src/daemon/host-transfer-proxy.ts +14 -18
  243. package/src/daemon/lifecycle.ts +0 -5
  244. package/src/daemon/mcp-reload-service.ts +5 -5
  245. package/src/daemon/message-types/sync.ts +1 -0
  246. package/src/daemon/orphan-reaper.ts +1 -1
  247. package/src/daemon/providers-setup.ts +1 -1
  248. package/src/daemon/reaction-history-render.ts +2 -1
  249. package/src/daemon/shutdown-handlers.ts +1 -9
  250. package/src/daemon/skill-memory-refresh.ts +64 -0
  251. package/src/daemon/tool-side-effects.ts +7 -18
  252. package/src/desktop/__tests__/fake-desktop.ts +122 -0
  253. package/src/desktop/desktop-chrome-policy.test.ts +79 -0
  254. package/src/desktop/desktop-chrome-policy.ts +47 -0
  255. package/src/desktop/desktop-chrome-session.ts +37 -0
  256. package/src/desktop/desktop-dependencies.test.ts +102 -0
  257. package/src/desktop/desktop-dependencies.ts +308 -0
  258. package/src/desktop/desktop-feature.test.ts +18 -0
  259. package/src/desktop/desktop-feature.ts +19 -0
  260. package/src/desktop/desktop-panel-config.test.ts +135 -0
  261. package/src/desktop/desktop-panel-config.ts +149 -0
  262. package/src/desktop/desktop-session-manager.test.ts +730 -0
  263. package/src/desktop/desktop-session-manager.ts +850 -0
  264. package/src/desktop/desktop-stream-bridge.test.ts +395 -0
  265. package/src/desktop/desktop-stream-bridge.ts +214 -0
  266. package/src/desktop/desktop-wallpaper.test.ts +191 -0
  267. package/src/desktop/desktop-wallpaper.ts +113 -0
  268. package/src/desktop/desktop-window-drag.ts +177 -0
  269. package/src/desktop/desktop-window-manager-config.test.ts +230 -0
  270. package/src/desktop/desktop-window-manager-config.ts +238 -0
  271. package/src/desktop/desktop-window-theme.ts +170 -0
  272. package/src/heartbeat/heartbeat-service.ts +21 -3
  273. package/src/home/__tests__/feed-writer.test.ts +22 -0
  274. package/src/home/feed-writer.ts +11 -2
  275. package/src/hooks/types.ts +33 -0
  276. package/src/i18n/AGENTS.md +22 -0
  277. package/src/i18n/__tests__/i18n.test.ts +108 -0
  278. package/src/i18n/index.ts +16 -0
  279. package/src/i18n/locales.ts +93 -0
  280. package/src/i18n/messages.ts +61 -0
  281. package/src/i18n/title.ts +25 -0
  282. package/src/inbound/platform-callback-registration.ts +4 -5
  283. package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
  284. package/src/live-voice/__tests__/live-voice-progress.test.ts +15 -15
  285. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
  286. package/src/live-voice/__tests__/live-voice-vad.test.ts +2 -2
  287. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
  288. package/src/live-voice/live-voice-photo.ts +25 -6
  289. package/src/live-voice/live-voice-session.ts +213 -693
  290. package/src/live-voice/protocol.ts +0 -2
  291. package/src/mcp/__tests__/effective-config.test.ts +2 -2
  292. package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
  293. package/src/mcp/effective-config.ts +3 -3
  294. package/src/mcp/mcp-header-store.ts +29 -45
  295. package/src/mcp/spec-schema.ts +111 -0
  296. package/src/mcp/startup.ts +8 -4
  297. package/src/mcp/workspace-mcp-config.ts +188 -0
  298. package/src/messaging/provider.ts +10 -3
  299. package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
  300. package/src/messaging/providers/callback-routing.ts +18 -0
  301. package/src/messaging/providers/channel-transport.ts +56 -0
  302. package/src/messaging/providers/discord/transport.ts +17 -0
  303. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
  304. package/src/messaging/providers/slack/adapter.ts +7 -44
  305. package/src/messaging/providers/slack/transport.ts +16 -0
  306. package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
  307. package/src/messaging/providers/telegram-bot/client.ts +0 -26
  308. package/src/messaging/providers/telegram-bot/transport.ts +20 -0
  309. package/src/messaging/providers/whatsapp/adapter.ts +1 -48
  310. package/src/messaging/providers/whatsapp/transport.ts +18 -0
  311. package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
  312. package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
  313. package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
  314. package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
  315. package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
  316. package/src/notifications/adapters/macos.ts +3 -0
  317. package/src/notifications/assistant-reply-producer.ts +8 -3
  318. package/src/notifications/conversation-pairing.ts +36 -16
  319. package/src/notifications/delivered-post-record.ts +11 -1
  320. package/src/notifications/guardian-feed-projection.ts +125 -5
  321. package/src/notifications/home-feed-side-effect.ts +55 -7
  322. package/src/notifications/schedule-result-producer.ts +66 -22
  323. package/src/oauth/AGENTS.md +2 -0
  324. package/src/oauth/__tests__/seed-providers-managed.test.ts +54 -0
  325. package/src/oauth/byo-connection.test.ts +105 -0
  326. package/src/oauth/byo-connection.ts +36 -6
  327. package/src/oauth/connection-resolver.test.ts +73 -0
  328. package/src/oauth/connection-resolver.ts +95 -2
  329. package/src/oauth/scope-utils.ts +24 -0
  330. package/src/oauth/seed-providers.ts +204 -25
  331. package/src/outbound-proxy/certs.ts +3 -5
  332. package/src/permissions/checker.test.ts +11 -6
  333. package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
  334. package/src/persistence/conversation-crud.ts +71 -63
  335. package/src/persistence/conversation-lineage.ts +12 -4
  336. package/src/persistence/conversation-queries.ts +2 -2
  337. package/src/persistence/conversation-title-placeholders.ts +23 -0
  338. package/src/persistence/conversation-title-service.ts +9 -26
  339. package/src/persistence/embeddings/qdrant-manager.ts +163 -18
  340. package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
  341. package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
  342. package/src/persistence/message-cursor.ts +78 -0
  343. package/src/persistence/message-reads.ts +30 -0
  344. package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
  345. package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
  346. package/src/persistence/schema/memory-core.ts +5 -0
  347. package/src/platform/client.test.ts +4 -2
  348. package/src/platform/client.ts +2 -10
  349. package/src/plugin-api/constants.ts +2 -0
  350. package/src/plugin-api/index.ts +1 -0
  351. package/src/plugin-api/types.ts +2 -0
  352. package/src/plugins/defaults/index.ts +2 -0
  353. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
  354. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
  355. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
  356. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
  357. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +20 -10
  358. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +5 -0
  359. package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
  360. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
  361. package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
  362. package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
  363. package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -2
  364. package/src/plugins/defaults/memory/context-search/format.ts +3 -2
  365. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +6 -3
  366. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +3 -2
  367. package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -7
  368. package/src/plugins/defaults/memory/hooks/init.ts +10 -0
  369. package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
  370. package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
  371. package/src/plugins/defaults/memory/host-utils.ts +10 -0
  372. package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
  373. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
  374. package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
  375. package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
  376. package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
  377. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
  378. package/src/plugins/defaults/memory/memory-retrospective-job.ts +12 -3
  379. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  380. package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
  381. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
  382. package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
  383. package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
  384. package/src/plugins/defaults/memory/startup.ts +28 -2
  385. package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
  386. package/src/plugins/defaults/memory/substrate/page-index.ts +6 -1
  387. package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
  388. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
  389. package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -2
  390. package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
  391. package/src/plugins/defaults/memory/v2/reranker.ts +6 -1
  392. package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
  393. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
  394. package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
  395. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +30 -0
  396. package/src/plugins/defaults/memory/v3/card.ts +3 -2
  397. package/src/plugins/defaults/memory/v3/pool-select.ts +5 -3
  398. package/src/plugins/defaults/memory/v3/sections.ts +9 -3
  399. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +5 -2
  400. package/src/plugins/defaults/memory/worker.ts +6 -0
  401. package/src/plugins/mcp-servers.ts +15 -75
  402. package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
  403. package/src/providers/__tests__/retry-callsite.test.ts +41 -0
  404. package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
  405. package/src/providers/anthropic/client.ts +0 -47
  406. package/src/providers/gemini/client.ts +9 -23
  407. package/src/providers/gemini-thought-signature.test.ts +63 -0
  408. package/src/providers/gemini-thought-signature.ts +44 -0
  409. package/src/providers/inference/__tests__/endpoint-probe.test.ts +42 -3
  410. package/src/providers/inference/auth.ts +2 -3
  411. package/src/providers/inference/connection-availability.ts +27 -8
  412. package/src/providers/inference/endpoint-probe.ts +10 -1
  413. package/src/providers/inference/resolve-auth.ts +22 -4
  414. package/src/providers/inference/service-account-token.ts +248 -0
  415. package/src/providers/model-catalog.ts +34 -0
  416. package/src/providers/model-intents.ts +10 -5
  417. package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
  418. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
  419. package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
  420. package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
  421. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
  422. package/src/providers/openai/chat-completions-provider.ts +212 -79
  423. package/src/providers/openai/codex-models.ts +0 -4
  424. package/src/providers/openai/google-thought-signature.ts +320 -0
  425. package/src/providers/opencode/client.test.ts +73 -4
  426. package/src/providers/opencode/client.ts +60 -3
  427. package/src/providers/openrouter/client.ts +20 -0
  428. package/src/providers/outbound-request-sanitize.ts +96 -0
  429. package/src/providers/retry.ts +30 -19
  430. package/src/providers/unparseable-tool-args.ts +5 -1
  431. package/src/runtime/AGENTS.md +3 -1
  432. package/src/runtime/__tests__/channel-send.test.ts +443 -0
  433. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +89 -0
  434. package/src/runtime/__tests__/host-observe.test.ts +40 -1
  435. package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
  436. package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
  437. package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
  438. package/src/runtime/agent-wake.ts +2 -1
  439. package/src/runtime/assistant-event-hub.ts +21 -3
  440. package/src/runtime/auth/same-actor.ts +38 -3
  441. package/src/runtime/channel-send.ts +316 -0
  442. package/src/runtime/host-observe.ts +11 -1
  443. package/src/runtime/http-server.ts +160 -151
  444. package/src/runtime/routes/__tests__/acp-routes.test.ts +159 -4
  445. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +205 -0
  446. package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
  447. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
  448. package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
  449. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
  450. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
  451. package/src/runtime/routes/acp-routes.test.ts +15 -0
  452. package/src/runtime/routes/acp-routes.ts +72 -9
  453. package/src/runtime/routes/app-management-routes.ts +73 -0
  454. package/src/runtime/routes/app-routes.ts +1 -1
  455. package/src/runtime/routes/channel-send-routes.ts +85 -0
  456. package/src/runtime/routes/conversation-list-routes.ts +14 -3
  457. package/src/runtime/routes/conversation-query-routes.ts +40 -0
  458. package/src/runtime/routes/conversation-routes.ts +16 -3
  459. package/src/runtime/routes/credential-routes.ts +19 -10
  460. package/src/runtime/routes/desktop-setup-routes.test.ts +74 -0
  461. package/src/runtime/routes/desktop-setup-routes.ts +35 -0
  462. package/src/runtime/routes/diagnostics-routes.ts +61 -4
  463. package/src/runtime/routes/events-routes.ts +4 -0
  464. package/src/runtime/routes/host-cu-routes.ts +6 -0
  465. package/src/runtime/routes/index.ts +4 -2
  466. package/src/runtime/routes/log-export-routes.ts +2 -0
  467. package/src/runtime/routes/mcp-auth-routes.ts +39 -47
  468. package/src/runtime/routes/migration-routes.ts +6 -20
  469. package/src/runtime/routes/oauth-commands-routes.ts +36 -0
  470. package/src/runtime/routes/platform-routes.ts +18 -15
  471. package/src/runtime/routes/workspace-routes.test.ts +55 -26
  472. package/src/runtime/routes/workspace-routes.ts +46 -63
  473. package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
  474. package/src/runtime/routes/workspace-tree-walk.ts +240 -0
  475. package/src/runtime/routes/workspace-utils.ts +17 -0
  476. package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
  477. package/src/runtime/services/conversation-serializer.ts +16 -3
  478. package/src/runtime/sync/resource-sync-events.ts +2 -1
  479. package/src/schedule/run-script.ts +2 -1
  480. package/src/schedule/schedule-store.ts +9 -2
  481. package/src/schedule/worker.ts +1 -1
  482. package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
  483. package/src/security/ces-http-record-backend.ts +120 -0
  484. package/src/security/ces-rpc-record-backend.ts +7 -0
  485. package/src/security/untrusted-content.ts +3 -1
  486. package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
  487. package/src/skills/workspace-skill-md-signature.ts +50 -0
  488. package/src/subagent/consult-context.ts +12 -25
  489. package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
  490. package/src/tools/acp/list-agents.test.ts +3 -1
  491. package/src/tools/acp/spawn.test.ts +146 -11
  492. package/src/tools/acp/spawn.ts +21 -3
  493. package/src/tools/acp/status.test.ts +94 -0
  494. package/src/tools/acp/status.ts +40 -3
  495. package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
  496. package/src/tools/browser/browser-execution.ts +1 -1
  497. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
  498. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
  499. package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
  500. package/src/tools/browser/network-recorder.ts +2 -1
  501. package/src/tools/capability-offer.test.ts +22 -2
  502. package/src/tools/capability-offer.ts +16 -1
  503. package/src/tools/computer-use/definitions.ts +121 -2
  504. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
  505. package/src/tools/credentials/metadata-store.ts +100 -0
  506. package/src/tools/filesystem/search.ts +2 -1
  507. package/src/tools/host-filesystem/edit.test.ts +2 -1
  508. package/src/tools/host-filesystem/edit.ts +2 -2
  509. package/src/tools/host-filesystem/read.test.ts +2 -1
  510. package/src/tools/host-filesystem/read.ts +2 -2
  511. package/src/tools/host-filesystem/transfer.test.ts +2 -1
  512. package/src/tools/host-filesystem/transfer.ts +2 -2
  513. package/src/tools/host-filesystem/write.test.ts +2 -1
  514. package/src/tools/host-filesystem/write.ts +2 -2
  515. package/src/tools/host-terminal/host-shell.test.ts +16 -0
  516. package/src/tools/host-terminal/host-shell.ts +4 -20
  517. package/src/tools/network/web-search.ts +6 -1
  518. package/src/tools/policy-context.ts +6 -7
  519. package/src/tools/skills/scaffold-managed.ts +61 -3
  520. package/src/tools/subagent/spawn.ts +5 -8
  521. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +57 -0
  522. package/src/tools/terminal/sanitized-bash.ts +105 -0
  523. package/src/tools/watch/watch-retro-report.ts +0 -8
  524. package/src/types/host-capabilities.ts +5 -3
  525. package/src/util/__tests__/language-subtag.test.ts +23 -1
  526. package/src/util/ca-bundle.ts +13 -0
  527. package/src/util/language-subtag.ts +16 -0
  528. package/src/util/platform.ts +8 -0
  529. package/src/util/unicode.ts +15 -1
  530. package/src/watch/watch-timeline.ts +2 -2
  531. package/src/workflows/run-manager.ts +2 -1
  532. package/src/workspace/git-porcelain.ts +66 -0
  533. package/src/workspace/git-service.ts +6 -55
  534. package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
  535. package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
  536. package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
  537. package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
  538. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
  539. package/src/workspace/migrations/registry.ts +6 -0
  540. package/src/__tests__/app-source-watcher.test.ts +0 -218
  541. package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
  542. package/src/__tests__/speaker-identification.test.ts +0 -56
  543. package/src/calls/speaker-identification.ts +0 -233
  544. package/src/daemon/app-source-watcher.ts +0 -218
  545. package/src/messaging/providers/whatsapp/client.ts +0 -29
  546. package/src/runtime/routes/debug-bash-routes.ts +0 -173
  547. package/src/util/fs-watcher-error.ts +0 -36
  548. /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
@@ -0,0 +1,423 @@
1
+ /**
2
+ * Progress narration cadence for one voice turn, shared by the live-voice
3
+ * session and the phone call controller.
4
+ *
5
+ * A voice turn that runs tools is silent while they run. The cadence speaks
6
+ * short audio-only progress updates into that dead air: a generated
7
+ * sentence about what the turn is doing (see `progress-narration.ts`), or a
8
+ * static fallback phrase when generation yields nothing. It follows the
9
+ * turn's tool activity rather than the clock: an "ops" burst, a long
10
+ * operation completing, or a full interval of audible silence with news
11
+ * since the last update, with `maxSilenceMs` as the heartbeat ceiling.
12
+ *
13
+ * The cadence owns the per-turn state (the tool-activity log, epochs,
14
+ * spacing, the self-re-arming idle timer). Everything transport-specific
15
+ * comes in through {@link ProgressCadenceHost}: whether the turn's audio is
16
+ * idle, when it last reached the listener, and how to speak a phrase.
17
+ */
18
+
19
+ import type { VoiceProgressConfig } from "../config/schemas/voice.js";
20
+ import { fixedPhraseLanguage } from "../util/language-subtag.js";
21
+ import { getLogger } from "../util/logger.js";
22
+ import type {
23
+ VoiceProgressNarrator,
24
+ VoiceProgressTextInput,
25
+ } from "./progress-narration.js";
26
+ import {
27
+ pickProgressPhrase,
28
+ PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
29
+ } from "./progress-phrases.js";
30
+
31
+ const log = getLogger("voice-progress-cadence");
32
+
33
+ /**
34
+ * One tool operation observed on a turn, fed by the bridge's structured tool
35
+ * callbacks. `completedAtMs`/`isError`/`resultPreview` land with tool_result.
36
+ */
37
+ export interface TurnProgressOp {
38
+ toolName: string;
39
+ toolUseId?: string;
40
+ startedAtMs: number;
41
+ completedAtMs?: number;
42
+ isError?: boolean;
43
+ resultPreview?: string;
44
+ }
45
+
46
+ /** What prompted a narration attempt. */
47
+ export type ProgressNarrationTrigger = "ops" | "idle" | "op_complete";
48
+
49
+ /** A tool completion as the bridge reports it. */
50
+ export interface ProgressToolResult {
51
+ toolName: string;
52
+ toolUseId?: string;
53
+ isError?: boolean;
54
+ resultPreview?: string;
55
+ }
56
+
57
+ /**
58
+ * Newest incomplete op, optionally restricted to a tool name (the
59
+ * tool_result fallback match when no toolUseId correlates).
60
+ */
61
+ export function findLastIncompleteOp(
62
+ ops: readonly TurnProgressOp[],
63
+ toolName?: string,
64
+ ): TurnProgressOp | undefined {
65
+ for (let i = ops.length - 1; i >= 0; i -= 1) {
66
+ const op = ops[i];
67
+ if (
68
+ op.completedAtMs === undefined &&
69
+ (toolName === undefined || op.toolName === toolName)
70
+ ) {
71
+ return op;
72
+ }
73
+ }
74
+ return undefined;
75
+ }
76
+
77
+ /**
78
+ * The transport-specific half of progress narration, supplied by the driver
79
+ * that owns the turn's audio.
80
+ */
81
+ export interface ProgressCadenceHost {
82
+ /** Identifies the turn in logs. */
83
+ turnId: string;
84
+ /** Dispatch timestamp of the turn (`Date.now()` at launch). */
85
+ launchedAtMs: number;
86
+ /** Aborts an in-flight narration generation with the turn. */
87
+ signal: AbortSignal;
88
+ /**
89
+ * Whether the turn may still be narrated: it is the live turn, it has not
90
+ * completed, and it is not waiting on a decision (a turn waiting on an
91
+ * approval says so once and is quiet after that).
92
+ */
93
+ canNarrate(): boolean;
94
+ /**
95
+ * No assistant audio is buffered, queued, in flight, or (estimatedly)
96
+ * still playing to the listener.
97
+ */
98
+ isAudioIdle(): boolean;
99
+ /**
100
+ * Wall-clock estimate of when audio already handed to the transport
101
+ * finishes reaching the listener, 0 when nothing is playing. Together with
102
+ * the cadence's own last-emitted stamp this anchors the dead-air countdown
103
+ * to when the listener last heard something.
104
+ */
105
+ playbackTailUntilMs(): number;
106
+ /** The caller's words this turn answers, for the narrator. */
107
+ transcriptSoFar(): string;
108
+ /** The turn's spoken language, when known. */
109
+ language(): string | undefined;
110
+ /**
111
+ * Bumped on every assistant text delta. A narration generated while the
112
+ * model was already speaking is stale and is dropped.
113
+ */
114
+ deltaEpoch(): number;
115
+ /**
116
+ * Speak one sanitized phrase, audio-only. `language` is the TTS hint the
117
+ * phrase must carry ("en" for a static fallback the localized table lacks
118
+ * in the turn's language); undefined rides the turn's language. Returns
119
+ * false when nothing was spoken.
120
+ */
121
+ speak(phrase: string, language: string | undefined): boolean;
122
+ /** Called after a narration was actually enqueued (metrics). */
123
+ onNarrationSpoken?(): void;
124
+ }
125
+
126
+ export interface ProgressCadenceOptions {
127
+ config: VoiceProgressConfig;
128
+ /** Null disables narration for the turn (no TTS, or narration off). */
129
+ narrator: VoiceProgressNarrator | null;
130
+ host: ProgressCadenceHost;
131
+ /**
132
+ * Rotates the static fallback phrase across turns so a session with
133
+ * repeated dead air does not repeat one sentence.
134
+ */
135
+ nextFallbackPhraseIndex: () => number;
136
+ }
137
+
138
+ export interface ProgressCadence {
139
+ /** Tool operations this turn, in start order. */
140
+ readonly ops: readonly TurnProgressOp[];
141
+ /** Narrations actually spoken this turn. */
142
+ readonly updatesSpoken: number;
143
+ /** A tool started: counts toward the "ops" trigger and is news for idle. */
144
+ toolStarted(toolName: string, toolUseId?: string): void;
145
+ /**
146
+ * A tool finished. A long operation completing narrates on its own beat;
147
+ * a quick one only counts as news.
148
+ */
149
+ toolFinished(event: ProgressToolResult): void;
150
+ /** The turn's audio last finished emitting now: restarts the dead-air clock. */
151
+ noteAudioSettled(): void;
152
+ /**
153
+ * Something else held the floor now (a spoken approval line, a bridge):
154
+ * narration keeps `minGapMs` from it.
155
+ */
156
+ noteFloorHolder(): void;
157
+ /**
158
+ * Arm (or re-arm) the dead-air timer. A no-op when narration is off for
159
+ * the turn.
160
+ */
161
+ arm(): void;
162
+ /** Stop the dead-air timer; the turn ended, was cancelled, or handed off. */
163
+ clear(): void;
164
+ }
165
+
166
+ export function createProgressCadence(
167
+ options: ProgressCadenceOptions,
168
+ ): ProgressCadence {
169
+ const { config, narrator, host, nextFallbackPhraseIndex } = options;
170
+ const ops: TurnProgressOp[] = [];
171
+ // Ops accumulated toward the next ops-triggered narration. Counted once
172
+ // per op, on start (not completion), so a burst of slow tools still trips
173
+ // the threshold while they run.
174
+ let opsSinceNarration = 0;
175
+ // Bumped by every observable change to the turn's tool activity. The idle
176
+ // trigger compares it against `narratedEpoch` so a tick with nothing new
177
+ // to report stays silent.
178
+ let stateEpoch = 0;
179
+ let narratedEpoch = 0;
180
+ let updatesSpoken = 0;
181
+ // When the last spoken floor-holder enqueued; gates `minGapMs`.
182
+ let lastFloorHolderAtMs: number | null = null;
183
+ // When the turn's audio last finished emitting (turn launch until anything
184
+ // speaks). Idle time is measured from when the listener last heard
185
+ // something, not from when the turn started.
186
+ let lastAudibleAtMs = Date.now();
187
+ let idleTimer: ReturnType<typeof setTimeout> | null = null;
188
+ // A narration generation is awaiting the narrator; serializes narrations.
189
+ let narrationInFlight = false;
190
+
191
+ const enabled = config.enabled && narrator !== null;
192
+
193
+ // When the turn's current audible silence began: the latest of the last
194
+ // emitted segment, the estimated playback end, and the last floor holder.
195
+ const silenceSinceMs = (): number =>
196
+ Math.max(
197
+ lastAudibleAtMs,
198
+ host.playbackTailUntilMs(),
199
+ lastFloorHolderAtMs ?? 0,
200
+ );
201
+
202
+ const canNarrate = (): boolean => host.canNarrate() && host.isAudioIdle();
203
+
204
+ const withinMinGap = (): boolean =>
205
+ lastFloorHolderAtMs !== null &&
206
+ Date.now() - lastFloorHolderAtMs < config.minGapMs;
207
+
208
+ // The idle tick has something worth saying when the turn's tool activity
209
+ // has moved since the last narration described it, or when the silence has
210
+ // run past `maxSilenceMs`, the heartbeat ceiling that proves the assistant
211
+ // is still alive on a turn with no observable activity at all.
212
+ const idleHasSomethingToSay = (): boolean => {
213
+ if (stateEpoch !== narratedEpoch) {
214
+ return true;
215
+ }
216
+ const silentForMs = Date.now() - silenceSinceMs();
217
+ if (silentForMs >= config.maxSilenceMs) {
218
+ return true;
219
+ }
220
+ log.debug(
221
+ { turnId: host.turnId, silentForMs },
222
+ "Voice progress narration held: nothing new since the last update",
223
+ );
224
+ return false;
225
+ };
226
+
227
+ const clearTimer = (): void => {
228
+ if (idleTimer !== null) {
229
+ clearTimeout(idleTimer);
230
+ idleTimer = null;
231
+ }
232
+ };
233
+
234
+ // Arms the dead-air timer. The countdown measures audible silence, not
235
+ // time since launch, so it covers mid-turn silences for the whole turn. On
236
+ // expiry with audio still pending, or with the silence not yet a full
237
+ // interval old, it re-arms for the remainder; only a full interval of
238
+ // audible silence reaches the gatekeeper. The interval is a polling
239
+ // cadence, not a speaking cadence: most ticks find nothing new and stay
240
+ // quiet.
241
+ const armTimer = (delayMs?: number): void => {
242
+ clearTimer();
243
+ idleTimer = setTimeout(() => {
244
+ idleTimer = null;
245
+ if (!host.canNarrate()) {
246
+ return;
247
+ }
248
+ if (!host.isAudioIdle()) {
249
+ armTimer();
250
+ return;
251
+ }
252
+ const remaining = silenceSinceMs() + config.idleIntervalMs - Date.now();
253
+ if (remaining > 0) {
254
+ armTimer(remaining);
255
+ return;
256
+ }
257
+ maybeNarrate("idle");
258
+ armTimer();
259
+ }, delayMs ?? config.idleIntervalMs);
260
+ };
261
+
262
+ // Generate and speak one audio-only progress narration. On a null result,
263
+ // idle narration falls back to a static phrase while activity-triggered
264
+ // narration stays silent.
265
+ const speakUpdate = async (
266
+ activeNarrator: VoiceProgressNarrator,
267
+ trigger: ProgressNarrationTrigger,
268
+ ): Promise<void> => {
269
+ narrationInFlight = true;
270
+ const deltaEpochAtLaunch = host.deltaEpoch();
271
+ // The activity this update describes. Tool events that land
272
+ // mid-generation are news the generated text cannot carry, so they must
273
+ // leave the idle trigger armed rather than count as already narrated.
274
+ const stateEpochAtLaunch = stateEpoch;
275
+ try {
276
+ const now = Date.now();
277
+ const currentOp = findLastIncompleteOp(ops);
278
+ const language = host.language();
279
+ const input: VoiceProgressTextInput = {
280
+ transcriptSoFar: host.transcriptSoFar(),
281
+ completedOps: ops
282
+ .filter(
283
+ (op): op is TurnProgressOp & { completedAtMs: number } =>
284
+ op.completedAtMs !== undefined,
285
+ )
286
+ // `ops` is in start order; the narrator wants completion order,
287
+ // which differs when parallel tools finish out of order.
288
+ .sort((a, b) => a.completedAtMs - b.completedAtMs)
289
+ .map((op) => ({
290
+ toolName: op.toolName,
291
+ ...(op.isError !== undefined ? { isError: op.isError } : {}),
292
+ ...(op.resultPreview !== undefined
293
+ ? { resultPreview: op.resultPreview }
294
+ : {}),
295
+ })),
296
+ currentOp: currentOp
297
+ ? {
298
+ toolName: currentOp.toolName,
299
+ elapsedMs: now - currentOp.startedAtMs,
300
+ }
301
+ : null,
302
+ turnElapsedMs: now - host.launchedAtMs,
303
+ updateIndex: updatesSpoken + 1,
304
+ ...(language !== undefined ? { languageHint: language } : {}),
305
+ };
306
+ const generated = await activeNarrator
307
+ .generateProgressText(input, host.signal)
308
+ // The narrator contract never rejects. Keep test stubs fail-soft too.
309
+ .catch(() => null);
310
+ // Re-check liveness and staleness after the provider call.
311
+ if (
312
+ !canNarrate() ||
313
+ host.deltaEpoch() !== deltaEpochAtLaunch ||
314
+ withinMinGap()
315
+ ) {
316
+ return;
317
+ }
318
+ let raw = generated;
319
+ // Generated text is in the turn's language; only the static fallback
320
+ // comes from a localized table and may need the "en" override.
321
+ let fillerLanguage: string | undefined;
322
+ if (raw === null) {
323
+ if (trigger !== "idle") {
324
+ return;
325
+ }
326
+ raw = pickProgressPhrase(nextFallbackPhraseIndex(), language);
327
+ fillerLanguage = fixedPhraseLanguage(
328
+ PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
329
+ language,
330
+ );
331
+ }
332
+ if (!host.speak(raw, fillerLanguage)) {
333
+ return;
334
+ }
335
+ lastFloorHolderAtMs = Date.now();
336
+ opsSinceNarration = 0;
337
+ narratedEpoch = stateEpochAtLaunch;
338
+ updatesSpoken += 1;
339
+ host.onNarrationSpoken?.();
340
+ // Restart the dead-air countdown from this narration.
341
+ armTimer();
342
+ } finally {
343
+ narrationInFlight = false;
344
+ }
345
+ };
346
+
347
+ // Gatekeeper. Speaks only while the turn is audibly silent, with one
348
+ // generation at a time and the configured spacing.
349
+ const maybeNarrate = (trigger: ProgressNarrationTrigger): void => {
350
+ if (
351
+ !enabled ||
352
+ narrator === null ||
353
+ !canNarrate() ||
354
+ narrationInFlight ||
355
+ withinMinGap() ||
356
+ (trigger === "ops" && opsSinceNarration < config.opsThreshold) ||
357
+ (trigger === "idle" && !idleHasSomethingToSay())
358
+ ) {
359
+ return;
360
+ }
361
+ void speakUpdate(narrator, trigger);
362
+ };
363
+
364
+ return {
365
+ get ops() {
366
+ return ops;
367
+ },
368
+ get updatesSpoken() {
369
+ return updatesSpoken;
370
+ },
371
+ toolStarted(toolName, toolUseId) {
372
+ ops.push({
373
+ toolName,
374
+ ...(toolUseId !== undefined ? { toolUseId } : {}),
375
+ startedAtMs: Date.now(),
376
+ });
377
+ opsSinceNarration += 1;
378
+ stateEpoch += 1;
379
+ maybeNarrate("ops");
380
+ },
381
+ toolFinished(event) {
382
+ // Match by toolUseId when present; otherwise the last-started
383
+ // incomplete op with the same name (parallel same-name ops resolve
384
+ // newest-first).
385
+ const op =
386
+ (event.toolUseId !== undefined
387
+ ? ops.find((o) => o.toolUseId === event.toolUseId)
388
+ : undefined) ?? findLastIncompleteOp(ops, event.toolName);
389
+ // A long-running op finishing is the beat the listener has been
390
+ // waiting through: it narrates immediately rather than waiting for
391
+ // `opsThreshold` more ops, which on a one-slow-tool turn never arrive.
392
+ // Short ops stay on the ops trigger.
393
+ let trigger: ProgressNarrationTrigger = "ops";
394
+ if (op) {
395
+ op.completedAtMs = Date.now();
396
+ if (event.isError !== undefined) {
397
+ op.isError = event.isError;
398
+ }
399
+ if (event.resultPreview !== undefined) {
400
+ op.resultPreview = event.resultPreview;
401
+ }
402
+ if (op.completedAtMs - op.startedAtMs >= config.longOpMs) {
403
+ trigger = "op_complete";
404
+ }
405
+ }
406
+ stateEpoch += 1;
407
+ maybeNarrate(trigger);
408
+ },
409
+ noteAudioSettled() {
410
+ lastAudibleAtMs = Date.now();
411
+ },
412
+ noteFloorHolder() {
413
+ lastFloorHolderAtMs = Date.now();
414
+ },
415
+ arm() {
416
+ if (!enabled) {
417
+ return;
418
+ }
419
+ armTimer();
420
+ },
421
+ clear: clearTimer,
422
+ };
423
+ }
@@ -188,6 +188,34 @@ function frontDoorRuleWithDigest(
188
188
  });
189
189
  }
190
190
 
191
+ /**
192
+ * The triage-and-escalate rule a turn's leg adds to its control prompt: the
193
+ * front-door leg decides and may hand off, the escalated leg continues the
194
+ * answer after a holding phrase was already spoken. Null when routing is off.
195
+ * One rule for both prompt shapes, the auto-built phone prompt and a
196
+ * caller-supplied one, so the front-door model is anchored to the same
197
+ * caller words on every transport.
198
+ */
199
+ function routingLegRuleFor(
200
+ opts: Pick<
201
+ VoiceTurnOptions,
202
+ "routingLeg" | "unifiedVerdict" | "spokenEscalationBridge"
203
+ >,
204
+ callerUtterance: string,
205
+ ): string | null {
206
+ switch (opts.routingLeg) {
207
+ case "front-door":
208
+ return frontDoorRuleWithDigest(
209
+ opts.unifiedVerdict === true,
210
+ callerUtterance,
211
+ );
212
+ case "escalated":
213
+ return escalatedContinuationRule(opts.spokenEscalationBridge);
214
+ default:
215
+ return null;
216
+ }
217
+ }
218
+
191
219
  /**
192
220
  * Exact message thrown when `opts.signal` aborts while the turn is waiting
193
221
  * for the conversation to become available. The call controller's abort
@@ -373,8 +401,9 @@ export interface VoiceTurnOptions {
373
401
  *
374
402
  * Deliberately separate from {@link userMessageInterface}: that field feeds
375
403
  * `resolveChannelCapabilities` and decides what the turn may do, so it is not
376
- * free to carry attribution. Absent for phone calls and for clients that send
377
- * no identity on the start frame.
404
+ * free to carry attribution. A phone call passes its call session id and a
405
+ * `phone_*` entry naming the call's direction, with no client. Absent for
406
+ * clients that send no identity on the start frame.
378
407
  */
379
408
  voiceTelemetry?: {
380
409
  sessionId: string;
@@ -447,7 +476,7 @@ export interface VoiceTurnOptions {
447
476
  onComplete?: () => void;
448
477
  /** Called when the agent loop encounters an error. */
449
478
  onError?: (message: string) => void;
450
- /** Event-name callbacks used by non-phone voice clients. */
479
+ /** Event-name callbacks: tool activity, persisted row ids, raw stream. */
451
480
  callbacks?: VoiceTurnCallbacks;
452
481
  /**
453
482
  * Called when this turn leaves a confirmation for the user to answer instead
@@ -482,10 +511,9 @@ export interface VoiceTurnOptions {
482
511
  */
483
512
  overrideProfile?: string;
484
513
  /**
485
- * Which leg of a triaged turn this is, so the auto-built phone control prompt
486
- * can add the front-door triage rule or the escalated continuation rule.
487
- * Undefined = routing off; no routing rules are added. Ignored when a caller
488
- * supplies its own `voiceControlPrompt`.
514
+ * Which leg of a triaged turn this is, so the control prompt (auto-built or
515
+ * caller-supplied) carries the front-door triage rule or the escalated
516
+ * continuation rule. Undefined = routing off; no routing rules are added.
489
517
  */
490
518
  routingLeg?: VoiceRoutingLeg;
491
519
  /**
@@ -611,9 +639,8 @@ function buildVoiceCallControlPrompt(opts: {
611
639
  task?: string | null;
612
640
  isCallerGuardian?: boolean;
613
641
  skipDisclosure?: boolean;
614
- routingLeg?: VoiceRoutingLeg;
615
- spokenEscalationBridge?: string;
616
- unifiedVerdict?: boolean;
642
+ /** The turn's routing-leg rule (see {@link routingLegRuleFor}), if any. */
643
+ routingLegRule?: string | null;
617
644
  }): string {
618
645
  const config = getConfig();
619
646
  const disclosureEnabled =
@@ -665,22 +692,18 @@ function buildVoiceCallControlPrompt(opts: {
665
692
  );
666
693
  }
667
694
 
668
- lines.push(
669
- '6. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
670
- );
671
-
672
695
  if (opts.isInbound) {
673
696
  if (opts.isCallerGuardian) {
674
697
  lines.push(
675
- '7. If the latest user turn is "(call connected — deliver opening greeting)", this is your user calling you. Answer casually and briefly, like picking up a call from someone you know well. For example: "Hey!" or "What\'s up?" Do NOT introduce yourself, do NOT say you are calling on behalf of anyone, and do NOT ask how you can help in a formal way. Keep it short and natural.',
698
+ '6. If the latest user turn is "(call connected — deliver opening greeting)", this is your user calling you. Answer casually and briefly, like picking up a call from someone you know well. For example: "Hey!" or "What\'s up?" Do NOT introduce yourself, do NOT say you are calling on behalf of anyone, and do NOT ask how you can help in a formal way. Keep it short and natural.',
676
699
  );
677
700
  } else {
678
701
  lines.push(
679
- '7. If the latest user turn is "(call connected — deliver opening greeting)", this is an inbound call you are answering (not a call you initiated). Greet the caller warmly and ask how you can help. Introduce yourself once at the start using your assistant name if you know it (for example: "Hey there, this is Ava, Sam\'s assistant. How can I help?"). If your assistant name is not known, skip the name and just identify yourself as the guardian\'s assistant. Never use a UUID-shaped internal assistant ID as your spoken name. Do NOT say "I\'m calling" or "I\'m calling on behalf of". Vary the wording; do not use a fixed template.',
702
+ '6. If the latest user turn is "(call connected — deliver opening greeting)", this is an inbound call you are answering (not a call you initiated). Greet the caller warmly and ask how you can help. Introduce yourself once at the start using your assistant name if you know it (for example: "Hey there, this is Ava, Sam\'s assistant. How can I help?"). If your assistant name is not known, skip the name and just identify yourself as the guardian\'s assistant. Never use a UUID-shaped internal assistant ID as your spoken name. Do NOT say "I\'m calling" or "I\'m calling on behalf of". Vary the wording; do not use a fixed template.',
680
703
  );
681
704
  }
682
705
  lines.push(
683
- "8. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.",
706
+ "7. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.",
684
707
  );
685
708
  } else {
686
709
  const disclosureReminder =
@@ -688,27 +711,22 @@ function buildVoiceCallControlPrompt(opts: {
688
711
  ? " However, the disclosure text from rule 0 is separate from self-introduction and must always be included in your opening greeting, even if the Task does not mention introducing yourself."
689
712
  : "";
690
713
  lines.push(
691
- '7. If the latest user turn is "(verification completed — transitioning into conversation)", the caller just completed a phone verification code challenge on this call. Greet them naturally and ask if there is anything you can help with. Keep it casual and brief.',
692
- `If the latest user turn is "(call connected — deliver opening greeting)", deliver your opening greeting based solely on the Task context above. The Task already describes how to open the call follow it directly without adding any extra introduction on top. If the Task says to introduce yourself, do so once. If the Task does not mention introducing yourself, skip the introduction.${disclosureReminder} Vary the wording naturally; do not use a fixed template.`,
693
- "8. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.",
714
+ '6. If the latest user turn is "(verification completed — transitioning into conversation)", the caller just completed a phone verification code challenge on this call. Greet them naturally and ask if there is anything you can help with. Keep it casual and brief.',
715
+ `If the latest user turn is "(call connected — deliver opening greeting)", deliver your opening greeting based solely on the Task context above. The Task already describes how to open the call, so follow it directly without adding any extra introduction on top. If the Task says to introduce yourself, do so once. If the Task does not mention introducing yourself, skip the introduction.${disclosureReminder} Vary the wording naturally; do not use a fixed template.`,
716
+ "7. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.",
694
717
  );
695
718
  }
696
719
 
697
720
  lines.push(
698
- "9. After the opening greeting turn, treat the Task field as background context only do not re-execute its instructions on subsequent turns.",
699
- '10. Do not make up information. If you are unsure, use [ASK_GUARDIAN: your question] to consult your guardian. For tool permission requests, use [ASK_GUARDIAN_APPROVAL: {"question":"...","toolName":"...","input":{...}}].',
700
- `11. ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Protocol markers like ${opts.isCallerGuardian ? "[END_CALL]" : "[ASK_GUARDIAN: ...] and [END_CALL]"} are not spoken text and should still be used normally.`,
701
- `12. Speak the caller's language: reply in the language of the caller's most recent actual speech, and follow them if they switch languages mid-call. Synthetic user turns (parenthetical markers like the call-connected and verification-completed notices) are not caller speech and never set the language. Before the caller has spoken, such as on the opening greeting turn, ${preSpeechLanguageRuleFragment(config.services.stt.language, sttCatalogKeyForRole(config.services.stt, "telephony"))}.`,
702
- `13. ${PHONE_NO_SETUP_FLOWS_RULE}`,
721
+ "8. After the opening greeting turn, treat the Task field as background context only. Do not re-execute its instructions on subsequent turns.",
722
+ '9. Do not make up information. If you are unsure, use [ASK_GUARDIAN: your question] to consult your guardian. For tool permission requests, use [ASK_GUARDIAN_APPROVAL: {"question":"...","toolName":"...","input":{...}}].',
723
+ `10. ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Protocol markers like ${opts.isCallerGuardian ? "[END_CALL]" : "[ASK_GUARDIAN: ...] and [END_CALL]"} are not spoken text and should still be used normally.`,
724
+ `11. Speak the caller's language: reply in the language of the caller's most recent actual speech, and follow them if they switch languages mid-call. Synthetic user turns (parenthetical markers like the call-connected and verification-completed notices) are not caller speech and never set the language. Before the caller has spoken, such as on the opening greeting turn, ${preSpeechLanguageRuleFragment(config.services.stt.language, sttCatalogKeyForRole(config.services.stt, "telephony"))}.`,
725
+ `12. ${PHONE_NO_SETUP_FLOWS_RULE}`,
703
726
  );
704
727
 
705
- // Triage-and-escalate routing rules. The front-door leg decides and may
706
- // hand off; the escalated leg continues the answer after a holding phrase
707
- // was already spoken.
708
- if (opts.routingLeg === "front-door") {
709
- lines.push(`14. ${frontDoorRuleWithDigest(opts.unifiedVerdict === true)}`);
710
- } else if (opts.routingLeg === "escalated") {
711
- lines.push(`14. ${escalatedContinuationRule(opts.spokenEscalationBridge)}`);
728
+ if (opts.routingLegRule) {
729
+ lines.push(`13. ${opts.routingLegRule}`);
712
730
  }
713
731
 
714
732
  lines.push("</voice_call_control>");
@@ -956,6 +974,10 @@ export async function startVoiceTurn(
956
974
  // control markers (ASK_GUARDIAN, END_CALL, etc.) and recognize opener turns.
957
975
  const isCallerGuardian = opts.trustContext?.trustClass === "guardian";
958
976
 
977
+ // The front-door rule anchors on the words the model sees as the user
978
+ // turn, which for a phone sentinel is its neutral persisted form rather
979
+ // than the marker itself.
980
+ const routingLegRule = routingLegRuleFor(opts, persistedContent);
959
981
  let voiceCallControlPrompt: string | null;
960
982
  if (opts.voiceControlPrompt === undefined) {
961
983
  voiceCallControlPrompt = buildVoiceCallControlPrompt({
@@ -963,23 +985,14 @@ export async function startVoiceTurn(
963
985
  task: opts.task,
964
986
  isCallerGuardian,
965
987
  skipDisclosure: opts.skipDisclosure,
966
- routingLeg: opts.routingLeg,
967
- spokenEscalationBridge: opts.spokenEscalationBridge,
968
- unifiedVerdict: opts.unifiedVerdict,
988
+ routingLegRule,
969
989
  });
970
990
  } else {
971
- // A caller-supplied prompt (e.g. live-voice) bypasses
972
- // buildVoiceCallControlPrompt, which is where the triage-and-escalate rule
973
- // is normally injected from `routingLeg`. Append it here too without it
974
- // the front-door leg would run on the fast profile but never learn the
975
- // verdict protocol, so it could not hold or hand off to the escalated leg.
991
+ // A caller-supplied prompt (live voice) replaces the phone prompt, so the
992
+ // routing-leg rule is appended to it here. Without it the front-door leg
993
+ // would run on the fast profile but never learn the verdict protocol, so
994
+ // it could not hold or hand off to the escalated leg.
976
995
  voiceCallControlPrompt = opts.voiceControlPrompt;
977
- const routingLegRule =
978
- opts.routingLeg === "front-door"
979
- ? frontDoorRuleWithDigest(opts.unifiedVerdict === true, opts.content)
980
- : opts.routingLeg === "escalated"
981
- ? escalatedContinuationRule(opts.spokenEscalationBridge)
982
- : null;
983
996
  if (voiceCallControlPrompt != null && routingLegRule) {
984
997
  voiceCallControlPrompt = `${voiceCallControlPrompt}\n\n${routingLegRule}`;
985
998
  }