@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (859) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +5 -1
  3. package/ARCHITECTURE.md +20 -21
  4. package/Dockerfile +7 -7
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +11 -2
  7. package/docs/architecture/turn-actor.md +15 -8
  8. package/docs/credential-execution-service.md +1 -1
  9. package/docs/desktop-browser-cli.md +35 -0
  10. package/docs/guardian-request-flow.md +16 -0
  11. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  12. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  13. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  14. package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
  15. package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
  16. package/openapi.yaml +564 -111
  17. package/package.json +7 -4
  18. package/scripts/bundled-plugin-packages.ts +154 -0
  19. package/scripts/generate-bundled-plugin-packages.ts +29 -0
  20. package/scripts/postinstall.ts +33 -0
  21. package/scripts/smoke-desktop-browser-cli.ts +280 -0
  22. package/scripts/smoke-desktop-window-theme.ts +186 -0
  23. package/scripts/sync-llm-catalog.ts +3 -0
  24. package/scripts/test.ts +15 -13
  25. package/scripts/voice-ttft-spike.ts +2 -2
  26. package/src/__tests__/agent-loop.test.ts +124 -0
  27. package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
  28. package/src/__tests__/anthropic-provider.test.ts +0 -63
  29. package/src/__tests__/app-compiler.test.ts +6 -0
  30. package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
  31. package/src/__tests__/app-dir-path-guard.test.ts +1 -3
  32. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  33. package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
  34. package/src/__tests__/assistant-event-hub.test.ts +44 -0
  35. package/src/__tests__/attachments-store.test.ts +22 -3
  36. package/src/__tests__/channel-approval.test.ts +9 -14
  37. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  38. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  39. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  40. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  41. package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
  42. package/src/__tests__/computer-use-tools.test.ts +19 -7
  43. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  44. package/src/__tests__/config-schema.test.ts +29 -30
  45. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  46. package/src/__tests__/config-watcher.test.ts +23 -8
  47. package/src/__tests__/connection-model-compat.test.ts +2 -2
  48. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  49. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  50. package/src/__tests__/conversation-agent-loop.test.ts +257 -6
  51. package/src/__tests__/conversation-attachments.test.ts +106 -0
  52. package/src/__tests__/conversation-error.test.ts +4 -1
  53. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  54. package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
  55. package/src/__tests__/conversation-pairing.test.ts +26 -0
  56. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
  57. package/src/__tests__/conversation-queue.test.ts +48 -207
  58. package/src/__tests__/conversation-retry-discard.test.ts +54 -1
  59. package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
  60. package/src/__tests__/conversation-skill-tools.test.ts +27 -0
  61. package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
  62. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
  63. package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
  64. package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
  65. package/src/__tests__/conversation-title-service.test.ts +14 -10
  66. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
  67. package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
  68. package/src/__tests__/credential-execution-client.test.ts +70 -53
  69. package/src/__tests__/credential-record-retire.test.ts +171 -1
  70. package/src/__tests__/credential-routes.test.ts +250 -3
  71. package/src/__tests__/credential-security-invariants.test.ts +5 -3
  72. package/src/__tests__/cu-unified-flow.test.ts +14 -4
  73. package/src/__tests__/db-conversation-tool-surface.test.ts +144 -0
  74. package/src/__tests__/document-sync-tags.test.ts +25 -0
  75. package/src/__tests__/drain-kick-guard.test.ts +0 -2
  76. package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
  77. package/src/__tests__/events-tail-route.test.ts +33 -0
  78. package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
  79. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  80. package/src/__tests__/git-porcelain.test.ts +47 -0
  81. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  82. package/src/__tests__/heartbeat-service.test.ts +56 -40
  83. package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
  84. package/src/__tests__/host-bash-proxy.test.ts +26 -45
  85. package/src/__tests__/host-browser-proxy.test.ts +6 -17
  86. package/src/__tests__/host-cu-proxy.test.ts +712 -18
  87. package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
  88. package/src/__tests__/host-file-proxy.test.ts +11 -23
  89. package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
  90. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  91. package/src/__tests__/list-all-apps.test.ts +40 -8
  92. package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
  93. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  94. package/src/__tests__/llm-catalog-parity.test.ts +51 -3
  95. package/src/__tests__/managed-store.test.ts +121 -0
  96. package/src/__tests__/mcp-auth-routes.test.ts +104 -8
  97. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  98. package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
  99. package/src/__tests__/mcp-health-check.test.ts +40 -43
  100. package/src/__tests__/mcp-list-plugin-servers.test.ts +85 -42
  101. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  102. package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
  103. package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
  104. package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
  105. package/src/__tests__/messaging-send-tool.test.ts +355 -548
  106. package/src/__tests__/model-intents.test.ts +3 -0
  107. package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
  108. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  109. package/src/__tests__/oauth-commands-routes.test.ts +282 -101
  110. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  111. package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
  112. package/src/__tests__/oauth-provider-serializer.test.ts +23 -0
  113. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  114. package/src/__tests__/openai-provider.test.ts +1 -0
  115. package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
  116. package/src/__tests__/persist-media-references.test.ts +50 -0
  117. package/src/__tests__/platform-callback-registration.test.ts +8 -16
  118. package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +13 -0
  120. package/src/__tests__/qdrant-manager.test.ts +132 -0
  121. package/src/__tests__/resolve-app.test.ts +135 -0
  122. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  123. package/src/__tests__/scaffold-managed-skill-tool.test.ts +317 -6
  124. package/src/__tests__/schedule-routes.test.ts +20 -0
  125. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  126. package/src/__tests__/script-proxy-certs.test.ts +35 -0
  127. package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
  128. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  129. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  130. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  131. package/src/__tests__/secure-keys.test.ts +7 -3
  132. package/src/__tests__/service-account-token.test.ts +247 -0
  133. package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
  134. package/src/__tests__/skill-feature-flags.test.ts +2 -2
  135. package/src/__tests__/skills.test.ts +20 -4
  136. package/src/__tests__/source-fingerprint.test.ts +184 -0
  137. package/src/__tests__/subagent-tool-gate-mode.test.ts +239 -3
  138. package/src/__tests__/subagent-tools.test.ts +3 -2
  139. package/src/__tests__/telephony-tts-guard.test.ts +38 -3
  140. package/src/__tests__/terminal-tools.test.ts +8 -0
  141. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  142. package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
  143. package/src/__tests__/ui-channel-variants.test.ts +1 -56
  144. package/src/__tests__/unicode.test.ts +49 -0
  145. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  146. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
  147. package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
  148. package/src/__tests__/workspace-git-service.test.ts +26 -0
  149. package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
  150. package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
  151. package/src/acp/__tests__/acp-credentials.test.ts +13 -0
  152. package/src/acp/__tests__/agent-process.test.ts +258 -4
  153. package/src/acp/__tests__/auth-required.test.ts +96 -0
  154. package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
  155. package/src/acp/__tests__/client-handler.test.ts +90 -2
  156. package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
  157. package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
  158. package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
  159. package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
  160. package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
  161. package/src/acp/__tests__/session-manager.test.ts +895 -4
  162. package/src/acp/acp-claude-oauth.ts +328 -14
  163. package/src/acp/acp-credentials.ts +19 -0
  164. package/src/acp/agent-process.test.ts +4 -1
  165. package/src/acp/agent-process.ts +96 -10
  166. package/src/acp/auth-required.ts +50 -1
  167. package/src/acp/auto-install.test.ts +9 -7
  168. package/src/acp/claude-token-refresh.ts +150 -0
  169. package/src/acp/client-handler.ts +16 -3
  170. package/src/acp/model-config.test.ts +166 -0
  171. package/src/acp/model-config.ts +138 -0
  172. package/src/acp/prepare-agent-env.ts +21 -6
  173. package/src/acp/resolve-agent.test.ts +220 -6
  174. package/src/acp/resolve-agent.ts +88 -23
  175. package/src/acp/session-manager.test.ts +196 -3
  176. package/src/acp/session-manager.ts +441 -18
  177. package/src/acp/session-snapshot.ts +260 -0
  178. package/src/acp/types.ts +50 -5
  179. package/src/agent/loop.ts +20 -0
  180. package/src/api/attachment-provenance.test.ts +68 -0
  181. package/src/api/computer-use-tool.test.ts +55 -0
  182. package/src/api/computer-use-tool.ts +27 -0
  183. package/src/api/events/acp-session-model-update.test.ts +88 -0
  184. package/src/api/events/acp-session-model-update.ts +46 -0
  185. package/src/api/events/app-files-changed.ts +4 -3
  186. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  187. package/src/api/events/desktop-activity-changed.ts +10 -0
  188. package/src/api/events/notification-intent.ts +2 -0
  189. package/src/api/events/user-message-echo.ts +7 -1
  190. package/src/api/index.ts +17 -0
  191. package/src/api/responses/conversation-message.ts +8 -0
  192. package/src/approvals/approval-primitive.ts +5 -2
  193. package/src/approvals/scoped-approval-grants.ts +6 -2
  194. package/src/apps/app-store.ts +7 -6
  195. package/src/apps/resolve-app.ts +81 -0
  196. package/src/apps/source-fingerprint.ts +266 -0
  197. package/src/avatar/identity-avatar.ts +7 -2
  198. package/src/browser/operations.ts +2 -1
  199. package/src/browser/types.ts +7 -0
  200. package/src/browser/virtual-desktop-target.ts +34 -0
  201. package/src/bundler/app-compiler.ts +15 -0
  202. package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
  203. package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
  204. package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
  205. package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
  206. package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
  207. package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
  208. package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
  209. package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
  210. package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +27 -4
  211. package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
  212. package/src/calls/__tests__/voice-control-protocol.test.ts +151 -27
  213. package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
  214. package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
  215. package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
  216. package/src/calls/__tests__/voice-session-bridge.test.ts +140 -3
  217. package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
  218. package/src/calls/barge-in-guard.ts +118 -0
  219. package/src/calls/call-constants.ts +24 -4
  220. package/src/calls/call-controller.ts +499 -199
  221. package/src/calls/call-transport.ts +16 -0
  222. package/src/calls/media-stream-output.ts +32 -0
  223. package/src/calls/media-stream-server.ts +82 -30
  224. package/src/calls/media-stream-stt-session.ts +15 -0
  225. package/src/{live-voice → calls}/progress-narration.ts +17 -4
  226. package/src/calls/resolve-call-tts-provider.ts +33 -2
  227. package/src/calls/tts-call-strategy.ts +9 -5
  228. package/src/calls/types.ts +2 -0
  229. package/src/calls/voice-control-protocol.ts +194 -21
  230. package/src/calls/voice-leg-coordinator.ts +189 -0
  231. package/src/calls/voice-progress-cadence.ts +423 -0
  232. package/src/calls/voice-session-bridge.ts +140 -76
  233. package/src/calls/voice-triage-escalate.ts +178 -59
  234. package/src/cli/commands/__tests__/apps.test.ts +245 -0
  235. package/src/cli/commands/__tests__/bash.test.ts +53 -0
  236. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  237. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  238. package/src/cli/commands/__tests__/plugins.test.ts +85 -0
  239. package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
  240. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  241. package/src/cli/commands/apps.help.ts +67 -13
  242. package/src/cli/commands/apps.ts +183 -0
  243. package/src/cli/commands/bash.help.ts +9 -13
  244. package/src/cli/commands/bash.ts +7 -25
  245. package/src/cli/commands/browser.help.ts +43 -0
  246. package/src/cli/commands/browser.ts +52 -8
  247. package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
  248. package/src/cli/commands/channels/index.help.ts +70 -2
  249. package/src/cli/commands/channels/index.ts +16 -3
  250. package/src/cli/commands/channels/send.ts +128 -0
  251. package/src/cli/commands/credentials.help.ts +9 -5
  252. package/src/cli/commands/credentials.ts +2 -2
  253. package/src/cli/commands/mcp.help.ts +8 -8
  254. package/src/cli/commands/mcp.ts +1 -1
  255. package/src/cli/commands/oauth/index.help.ts +10 -0
  256. package/src/cli/commands/oauth/providers.ts +12 -0
  257. package/src/cli/commands/oauth/request.test.ts +179 -2
  258. package/src/cli/commands/oauth/request.ts +41 -10
  259. package/src/cli/commands/oauth/status.ts +5 -0
  260. package/src/cli/commands/plugins.ts +73 -23
  261. package/src/cli/commands/roadmap.help.ts +5 -1
  262. package/src/cli/commands/roadmap.ts +1 -1
  263. package/src/cli/commands/schedules.ts +2 -0
  264. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  265. package/src/cli/lib/__tests__/install-from-github.test.ts +219 -0
  266. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  267. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  268. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +251 -0
  269. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +65 -2
  270. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
  271. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
  272. package/src/cli/lib/__tests__/plugin-details.test.ts +35 -2
  273. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
  274. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
  275. package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
  276. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  277. package/src/cli/lib/bundled-marketplace.json +962 -81
  278. package/src/cli/lib/bundled-plugin-packages.json +786 -0
  279. package/src/cli/lib/bundled-plugin-packages.ts +87 -0
  280. package/src/cli/lib/diff-plugin.ts +1 -1
  281. package/src/cli/lib/inspect-plugin.ts +87 -12
  282. package/src/cli/lib/install-from-github.ts +150 -94
  283. package/src/cli/lib/install-from-platform.ts +14 -9
  284. package/src/cli/lib/list-installed-plugins.ts +21 -35
  285. package/src/cli/lib/plugin-catalog-cache.ts +22 -3
  286. package/src/cli/lib/plugin-catalog-local.ts +13 -3
  287. package/src/cli/lib/plugin-catalog-platform.ts +6 -1
  288. package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
  289. package/src/cli/lib/plugin-details.ts +33 -8
  290. package/src/cli/lib/plugin-marketplace.ts +121 -21
  291. package/src/cli/lib/plugin-pin-history.ts +5 -2
  292. package/src/cli/lib/search-plugins.ts +58 -16
  293. package/src/cli/lib/uninstall-plugin.ts +59 -1
  294. package/src/cli/lib/upgrade-plugin.ts +61 -8
  295. package/src/cli/output.ts +18 -4
  296. package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
  297. package/src/config/__tests__/default-provider.test.ts +4 -0
  298. package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
  299. package/src/config/__tests__/platform-identity.test.ts +255 -0
  300. package/src/config/__tests__/platform-rehydration.test.ts +99 -9
  301. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  302. package/src/config/acp-agent-write.ts +43 -0
  303. package/src/config/acp-defaults.test.ts +13 -3
  304. package/src/config/acp-defaults.ts +23 -6
  305. package/src/config/acp-schema.test.ts +81 -0
  306. package/src/config/acp-schema.ts +30 -2
  307. package/src/config/bundled-skills/acp/SKILL.md +23 -15
  308. package/src/config/bundled-skills/acp/TOOLS.json +7 -3
  309. package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
  310. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
  311. package/src/config/bundled-skills/computer-use/SKILL.md +59 -2
  312. package/src/config/bundled-skills/computer-use/TOOLS.json +101 -2
  313. package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
  314. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
  315. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
  316. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
  317. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  318. package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
  319. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
  320. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
  321. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
  322. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
  323. package/src/config/bundled-skills/schedule/SKILL.md +7 -8
  324. package/src/config/bundled-skills/screen-annotation/SKILL.md +10 -3
  325. package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
  326. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  327. package/src/config/bundled-skills/skill-management/TOOLS.json +13 -9
  328. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
  329. package/src/config/default-profile-catalog.ts +1 -1
  330. package/src/config/env.ts +2 -2
  331. package/src/config/feature-flag-registry.json +33 -13
  332. package/src/config/loader.ts +11 -17
  333. package/src/config/memory-v3-gate.test.ts +31 -0
  334. package/src/config/memory-v3-gate.ts +20 -16
  335. package/src/config/platform-identity.ts +201 -0
  336. package/src/config/platform-rehydration.ts +11 -27
  337. package/src/config/profile-text-generation.test.ts +51 -0
  338. package/src/config/profile-text-generation.ts +51 -0
  339. package/src/config/schema.ts +2 -2
  340. package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
  341. package/src/config/schemas/__tests__/voice.test.ts +148 -0
  342. package/src/config/schemas/calls.ts +1 -28
  343. package/src/config/schemas/live-voice.ts +1 -173
  344. package/src/config/schemas/llm.ts +7 -2
  345. package/src/config/schemas/mcp.ts +15 -9
  346. package/src/config/schemas/memory-retrospective.ts +10 -0
  347. package/src/config/schemas/monitoring.ts +9 -0
  348. package/src/config/schemas/services.ts +18 -0
  349. package/src/config/schemas/skills.ts +0 -14
  350. package/src/config/schemas/voice.ts +150 -0
  351. package/src/config/skills.ts +13 -21
  352. package/src/credential-execution/ces-runtime.ts +2 -2
  353. package/src/credential-execution/executable-discovery.ts +36 -134
  354. package/src/credential-execution/process-manager.test.ts +37 -42
  355. package/src/credential-execution/process-manager.ts +24 -19
  356. package/src/credential-health/credential-health-service.ts +11 -17
  357. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  358. package/src/daemon/__tests__/conversation-tool-setup.test.ts +109 -121
  359. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +5 -1
  360. package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
  361. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  362. package/src/daemon/app-change-notify.ts +45 -0
  363. package/src/daemon/assistant-attachments.ts +20 -12
  364. package/src/daemon/config-watcher.ts +13 -194
  365. package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
  366. package/src/daemon/conversation-agent-loop.ts +12 -0
  367. package/src/daemon/conversation-attachments.ts +93 -12
  368. package/src/daemon/conversation-client-surface.ts +26 -0
  369. package/src/daemon/conversation-error.ts +3 -2
  370. package/src/daemon/conversation-media-retry.ts +2 -1
  371. package/src/daemon/conversation-process.ts +35 -64
  372. package/src/daemon/conversation-runtime-assembly.ts +21 -8
  373. package/src/daemon/conversation-skill-tools.ts +13 -31
  374. package/src/daemon/conversation-store.ts +1 -1
  375. package/src/daemon/conversation-surfaces.ts +21 -10
  376. package/src/daemon/conversation-tool-setup.ts +135 -109
  377. package/src/daemon/conversation-turn-finalize.ts +26 -13
  378. package/src/daemon/conversation.ts +17 -3
  379. package/src/daemon/daemon-control.ts +3 -7
  380. package/src/daemon/dictation-profile-store.ts +2 -1
  381. package/src/daemon/handlers/shared.ts +2 -1
  382. package/src/daemon/handlers/skills.ts +6 -3
  383. package/src/daemon/host-bash-proxy.ts +8 -12
  384. package/src/daemon/host-browser-proxy.ts +15 -1
  385. package/src/daemon/host-cu-proxy.ts +275 -45
  386. package/src/daemon/host-file-proxy.ts +8 -10
  387. package/src/daemon/host-proxy-preactivation.ts +4 -1
  388. package/src/daemon/host-transfer-proxy.ts +14 -18
  389. package/src/daemon/lifecycle.ts +4 -9
  390. package/src/daemon/mcp-reload-service.ts +13 -6
  391. package/src/daemon/message-types/shared.ts +1 -0
  392. package/src/daemon/message-types/sync.ts +2 -0
  393. package/src/daemon/orphan-reaper.ts +4 -3
  394. package/src/daemon/persist-media-references.ts +20 -3
  395. package/src/daemon/providers-setup.ts +5 -1
  396. package/src/daemon/reaction-history-render.ts +2 -1
  397. package/src/daemon/shutdown-handlers.ts +1 -9
  398. package/src/daemon/skill-memory-refresh.ts +64 -0
  399. package/src/daemon/tool-setup-types.ts +6 -0
  400. package/src/daemon/tool-side-effects.ts +7 -18
  401. package/src/daemon/wake-conversation-ops.ts +38 -15
  402. package/src/desktop/__tests__/fake-desktop.ts +123 -0
  403. package/src/desktop/desktop-automation-lease.test.ts +351 -0
  404. package/src/desktop/desktop-automation-lease.ts +306 -0
  405. package/src/desktop/desktop-browser-client.test.ts +413 -0
  406. package/src/desktop/desktop-browser-client.ts +430 -0
  407. package/src/desktop/desktop-browser-cursor.ts +40 -0
  408. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  409. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  410. package/src/desktop/desktop-browser-operations.ts +130 -0
  411. package/src/desktop/desktop-chrome-policy.test.ts +79 -0
  412. package/src/desktop/desktop-chrome-policy.ts +47 -0
  413. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  414. package/src/desktop/desktop-chrome-session.ts +87 -0
  415. package/src/desktop/desktop-dependencies.test.ts +131 -0
  416. package/src/desktop/desktop-dependencies.ts +329 -0
  417. package/src/desktop/desktop-display.ts +9 -0
  418. package/src/desktop/desktop-panel-config.test.ts +135 -0
  419. package/src/desktop/desktop-panel-config.ts +162 -0
  420. package/src/desktop/desktop-session-manager.test.ts +813 -0
  421. package/src/desktop/desktop-session-manager.ts +970 -0
  422. package/src/desktop/desktop-stream-bridge.test.ts +395 -0
  423. package/src/desktop/desktop-stream-bridge.ts +214 -0
  424. package/src/desktop/desktop-wallpaper.test.ts +191 -0
  425. package/src/desktop/desktop-wallpaper.ts +113 -0
  426. package/src/desktop/desktop-window-drag.ts +177 -0
  427. package/src/desktop/desktop-window-manager-config.test.ts +230 -0
  428. package/src/desktop/desktop-window-manager-config.ts +238 -0
  429. package/src/desktop/desktop-window-theme.ts +170 -0
  430. package/src/desktop/virtual-desktop-feature.ts +18 -0
  431. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  432. package/src/heartbeat/heartbeat-service.ts +21 -3
  433. package/src/home/__tests__/feed-writer.test.ts +22 -0
  434. package/src/home/feed-writer.ts +11 -2
  435. package/src/hooks/types.ts +33 -0
  436. package/src/i18n/AGENTS.md +22 -0
  437. package/src/i18n/__tests__/i18n.test.ts +112 -0
  438. package/src/i18n/index.ts +16 -0
  439. package/src/i18n/locales.ts +93 -0
  440. package/src/i18n/messages.ts +70 -0
  441. package/src/i18n/title.ts +25 -0
  442. package/src/inbound/platform-callback-registration.ts +4 -5
  443. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  444. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  445. package/src/ipc/assistant-server.ts +15 -1
  446. package/src/ipc/cli-client.ts +13 -5
  447. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
  448. package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
  449. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
  450. package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
  451. package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
  452. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  453. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
  454. package/src/live-voice/__tests__/live-voice-vad.test.ts +660 -5
  455. package/src/live-voice/__tests__/protocol.test.ts +95 -0
  456. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
  457. package/src/live-voice/__tests__/session-controls.test.ts +156 -0
  458. package/src/live-voice/live-voice-metrics.ts +9 -0
  459. package/src/live-voice/live-voice-photo.ts +25 -6
  460. package/src/live-voice/live-voice-session.ts +1081 -760
  461. package/src/live-voice/protocol.ts +89 -2
  462. package/src/live-voice/session-controls.ts +197 -0
  463. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  464. package/src/mcp/__tests__/effective-config.test.ts +13 -2
  465. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  466. package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
  467. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  468. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  469. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  470. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  471. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  472. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  473. package/src/mcp/__tests__/startup.test.ts +30 -7
  474. package/src/mcp/__tests__/tool-caps.test.ts +107 -0
  475. package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
  476. package/src/mcp/client.ts +64 -16
  477. package/src/mcp/credential-target.ts +105 -0
  478. package/src/mcp/effective-config.ts +45 -18
  479. package/src/mcp/manager.ts +201 -81
  480. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  481. package/src/mcp/mcp-auth-state.ts +38 -6
  482. package/src/mcp/mcp-header-store.ts +29 -45
  483. package/src/mcp/mcp-oauth-provider.ts +101 -33
  484. package/src/mcp/spec-schema.ts +111 -0
  485. package/src/mcp/startup.ts +41 -17
  486. package/src/mcp/tool-caps.ts +176 -0
  487. package/src/mcp/workspace-mcp-config.ts +188 -0
  488. package/src/messaging/provider-message-metadata.ts +3 -3
  489. package/src/messaging/provider.ts +10 -3
  490. package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
  491. package/src/messaging/providers/callback-routing.ts +18 -0
  492. package/src/messaging/providers/channel-transport.ts +56 -0
  493. package/src/messaging/providers/discord/transport.ts +17 -0
  494. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
  495. package/src/messaging/providers/slack/adapter.ts +7 -44
  496. package/src/messaging/providers/slack/transport.ts +16 -0
  497. package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
  498. package/src/messaging/providers/telegram-bot/client.ts +0 -26
  499. package/src/messaging/providers/telegram-bot/transport.ts +20 -0
  500. package/src/messaging/providers/whatsapp/adapter.ts +1 -48
  501. package/src/messaging/providers/whatsapp/transport.ts +18 -0
  502. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  503. package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
  504. package/src/monitoring/mount-watch.ts +348 -0
  505. package/src/monitoring/plugin-auto-update.ts +6 -0
  506. package/src/monitoring/worker.ts +9 -0
  507. package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
  508. package/src/notifications/__tests__/copy-composer.test.ts +70 -0
  509. package/src/notifications/__tests__/decision-engine.test.ts +175 -0
  510. package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
  511. package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
  512. package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
  513. package/src/notifications/adapters/macos.ts +3 -0
  514. package/src/notifications/assistant-reply-producer.ts +8 -3
  515. package/src/notifications/conversation-pairing.ts +36 -16
  516. package/src/notifications/copy-composer.ts +11 -3
  517. package/src/notifications/decision-engine.ts +43 -8
  518. package/src/notifications/delivered-post-record.ts +11 -1
  519. package/src/notifications/guardian-feed-projection.ts +125 -5
  520. package/src/notifications/home-feed-side-effect.ts +55 -7
  521. package/src/notifications/schedule-result-producer.ts +64 -22
  522. package/src/oauth/AGENTS.md +4 -0
  523. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  524. package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
  525. package/src/oauth/byo-connection.test.ts +105 -0
  526. package/src/oauth/byo-connection.ts +36 -6
  527. package/src/oauth/connection-resolver.test.ts +100 -0
  528. package/src/oauth/connection-resolver.ts +120 -3
  529. package/src/oauth/identity-verifier.ts +24 -0
  530. package/src/oauth/oauth-store.ts +11 -1
  531. package/src/oauth/provider-serializer.ts +10 -0
  532. package/src/oauth/scope-utils.ts +24 -0
  533. package/src/oauth/seed-providers.ts +367 -27
  534. package/src/onboarding/checkin-event.ts +2 -1
  535. package/src/outbound-proxy/certs.ts +3 -5
  536. package/src/permissions/checker.test.ts +11 -6
  537. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  538. package/src/permissions/confirmation-guardian-request.ts +2 -3
  539. package/src/persistence/attachments-store.ts +84 -44
  540. package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
  541. package/src/persistence/conversation-crud.ts +182 -83
  542. package/src/persistence/conversation-lineage.ts +12 -4
  543. package/src/persistence/conversation-plugin-facade.ts +13 -0
  544. package/src/persistence/conversation-queries.ts +2 -2
  545. package/src/persistence/conversation-title-placeholders.ts +23 -0
  546. package/src/persistence/conversation-title-service.ts +9 -26
  547. package/src/persistence/conversation-tool-surface.ts +86 -0
  548. package/src/persistence/conversation-types.test.ts +32 -0
  549. package/src/persistence/conversation-types.ts +26 -4
  550. package/src/persistence/embeddings/qdrant-manager.ts +163 -18
  551. package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
  552. package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
  553. package/src/persistence/message-cursor.ts +78 -0
  554. package/src/persistence/message-reads.ts +30 -0
  555. package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
  556. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  557. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  558. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  559. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  560. package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
  561. package/src/persistence/schema/conversation-tool-surfaces.ts +22 -0
  562. package/src/persistence/schema/index.ts +1 -0
  563. package/src/persistence/schema/memory-core.ts +5 -0
  564. package/src/persistence/schema/oauth.ts +1 -0
  565. package/src/persistence/steps.ts +15 -3
  566. package/src/platform/client.test.ts +4 -2
  567. package/src/platform/client.ts +2 -10
  568. package/src/plugin-api/constants.ts +2 -0
  569. package/src/plugin-api/conversation-turn.ts +31 -7
  570. package/src/plugin-api/index.ts +10 -1
  571. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  572. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  573. package/src/plugin-api/types.ts +2 -0
  574. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  575. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  576. package/src/plugins/defaults/index.ts +2 -0
  577. package/src/plugins/defaults/memory/AGENTS.md +14 -2
  578. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  579. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  580. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  581. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
  582. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
  583. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
  584. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
  585. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +66 -10
  586. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +20 -0
  587. package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
  588. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
  589. package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
  590. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  591. package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
  592. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  593. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  594. package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -3
  595. package/src/plugins/defaults/memory/context-search/format.ts +4 -2
  596. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +7 -3
  597. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +4 -2
  598. package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -8
  599. package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
  600. package/src/plugins/defaults/memory/hooks/init.ts +10 -0
  601. package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
  602. package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
  603. package/src/plugins/defaults/memory/injectors.ts +4 -3
  604. package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
  605. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
  606. package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
  607. package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
  608. package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
  609. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
  610. package/src/plugins/defaults/memory/memory-retrospective-job.ts +67 -184
  611. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  612. package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
  613. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
  614. package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
  615. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  616. package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
  617. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
  618. package/src/plugins/defaults/memory/startup.ts +28 -2
  619. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  620. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  621. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  622. package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
  623. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  624. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  625. package/src/plugins/defaults/memory/substrate/page-index.ts +7 -1
  626. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
  627. package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
  628. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  629. package/src/plugins/defaults/memory/tools.ts +1 -1
  630. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
  631. package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -1
  632. package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
  633. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  634. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  635. package/src/plugins/defaults/memory/v2/reranker.ts +7 -1
  636. package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
  637. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  638. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  639. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
  640. package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
  641. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +51 -0
  642. package/src/plugins/defaults/memory/v3/card.ts +4 -2
  643. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  644. package/src/plugins/defaults/memory/v3/orchestrate.ts +86 -22
  645. package/src/plugins/defaults/memory/v3/pool-select.ts +13 -8
  646. package/src/plugins/defaults/memory/v3/sections.ts +10 -3
  647. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +13 -1
  648. package/src/plugins/defaults/memory/worker.ts +6 -0
  649. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  650. package/src/plugins/external-plugin-loader.ts +49 -86
  651. package/src/plugins/installed-plugin-dirs.ts +6 -4
  652. package/src/plugins/mcp-servers.ts +24 -90
  653. package/src/plugins/mtime-cache.ts +2 -1
  654. package/src/plugins/user-loader.ts +2 -1
  655. package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
  656. package/src/providers/__tests__/retry-callsite.test.ts +41 -0
  657. package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
  658. package/src/providers/anthropic/client.ts +0 -47
  659. package/src/providers/gemini/client.ts +9 -23
  660. package/src/providers/gemini-thought-signature.test.ts +63 -0
  661. package/src/providers/gemini-thought-signature.ts +44 -0
  662. package/src/providers/inference/__tests__/endpoint-probe.test.ts +87 -3
  663. package/src/providers/inference/adapter-factory.ts +6 -0
  664. package/src/providers/inference/auth.ts +2 -3
  665. package/src/providers/inference/connection-availability.ts +27 -8
  666. package/src/providers/inference/endpoint-probe.ts +36 -12
  667. package/src/providers/inference/resolve-auth.ts +22 -4
  668. package/src/providers/inference/service-account-token.ts +248 -0
  669. package/src/providers/jev/client.test.ts +260 -0
  670. package/src/providers/jev/client.ts +518 -0
  671. package/src/providers/model-catalog.ts +92 -3
  672. package/src/providers/model-intents.ts +10 -5
  673. package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
  674. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
  675. package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
  676. package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
  677. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
  678. package/src/providers/openai/chat-completions-provider.ts +212 -79
  679. package/src/providers/openai/codex-models.ts +0 -4
  680. package/src/providers/openai/google-thought-signature.ts +320 -0
  681. package/src/providers/openai/responses-provider.ts +9 -2
  682. package/src/providers/opencode/client.test.ts +240 -4
  683. package/src/providers/opencode/client.ts +129 -3
  684. package/src/providers/openrouter/client.ts +20 -0
  685. package/src/providers/outbound-request-sanitize.ts +96 -0
  686. package/src/providers/retry.ts +30 -19
  687. package/src/providers/unparseable-tool-args.ts +5 -1
  688. package/src/runtime/AGENTS.md +9 -3
  689. package/src/runtime/__tests__/agent-wake.test.ts +86 -1
  690. package/src/runtime/__tests__/channel-send.test.ts +443 -0
  691. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +132 -0
  692. package/src/runtime/__tests__/host-observe.test.ts +40 -1
  693. package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
  694. package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
  695. package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
  696. package/src/runtime/agent-wake.ts +22 -5
  697. package/src/runtime/assistant-event-hub.ts +21 -3
  698. package/src/runtime/auth/same-actor.ts +38 -3
  699. package/src/runtime/channel-send.ts +316 -0
  700. package/src/runtime/guardian-action-service.ts +2 -17
  701. package/src/runtime/guardian-reply-router.ts +1 -8
  702. package/src/runtime/host-observe.ts +11 -1
  703. package/src/runtime/http-server.ts +160 -151
  704. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  705. package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
  706. package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
  707. package/src/runtime/routes/__tests__/acp-routes.test.ts +178 -4
  708. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
  709. package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
  710. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
  711. package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
  712. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
  713. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  714. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
  715. package/src/runtime/routes/__tests__/plugins-routes.test.ts +127 -3
  716. package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
  717. package/src/runtime/routes/acp-routes.test.ts +15 -0
  718. package/src/runtime/routes/acp-routes.ts +84 -165
  719. package/src/runtime/routes/app-management-routes.ts +90 -2
  720. package/src/runtime/routes/app-routes.ts +1 -1
  721. package/src/runtime/routes/browser-context.ts +39 -0
  722. package/src/runtime/routes/browser-routes.ts +16 -47
  723. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  724. package/src/runtime/routes/channel-route-shared.ts +1 -9
  725. package/src/runtime/routes/channel-send-routes.ts +85 -0
  726. package/src/runtime/routes/conversation-list-routes.ts +14 -3
  727. package/src/runtime/routes/conversation-query-routes.ts +40 -0
  728. package/src/runtime/routes/conversation-routes.ts +45 -7
  729. package/src/runtime/routes/credential-routes.ts +20 -14
  730. package/src/runtime/routes/desktop-setup-routes.test.ts +89 -0
  731. package/src/runtime/routes/desktop-setup-routes.ts +41 -0
  732. package/src/runtime/routes/diagnostics-routes.ts +61 -4
  733. package/src/runtime/routes/events-routes.ts +4 -0
  734. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  735. package/src/runtime/routes/host-cu-routes.ts +6 -0
  736. package/src/runtime/routes/inbound-message-handler.ts +2 -3
  737. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
  738. package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
  739. package/src/runtime/routes/index.ts +4 -2
  740. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  741. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  742. package/src/runtime/routes/log-export-routes.ts +2 -0
  743. package/src/runtime/routes/mcp-auth-routes.ts +141 -137
  744. package/src/runtime/routes/migration-routes.ts +6 -20
  745. package/src/runtime/routes/oauth-commands-routes.ts +98 -45
  746. package/src/runtime/routes/oauth-providers.ts +11 -0
  747. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  748. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  749. package/src/runtime/routes/platform-routes.ts +18 -15
  750. package/src/runtime/routes/plugins-routes.ts +83 -50
  751. package/src/runtime/routes/schedule-routes.ts +2 -0
  752. package/src/runtime/routes/secret-routes.ts +99 -119
  753. package/src/runtime/routes/workspace-routes.test.ts +55 -26
  754. package/src/runtime/routes/workspace-routes.ts +46 -63
  755. package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
  756. package/src/runtime/routes/workspace-tree-walk.ts +240 -0
  757. package/src/runtime/routes/workspace-utils.ts +17 -0
  758. package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
  759. package/src/runtime/services/conversation-serializer.ts +16 -3
  760. package/src/runtime/sync/resource-sync-events.ts +6 -1
  761. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  762. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  763. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  764. package/src/schedule/run-script.ts +4 -3
  765. package/src/schedule/schedule-store.ts +9 -2
  766. package/src/schedule/scheduler.ts +47 -11
  767. package/src/schedule/tool-surface-readiness.ts +65 -0
  768. package/src/schedule/worker-mcp.ts +85 -0
  769. package/src/schedule/worker.ts +9 -31
  770. package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
  771. package/src/security/ces-http-record-backend.ts +120 -0
  772. package/src/security/ces-rpc-record-backend.ts +7 -0
  773. package/src/security/secure-keys.ts +60 -48
  774. package/src/security/untrusted-content.ts +3 -1
  775. package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
  776. package/src/skills/managed-store.ts +153 -31
  777. package/src/skills/workspace-skill-md-signature.ts +50 -0
  778. package/src/subagent/consult-context.ts +12 -25
  779. package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
  780. package/src/tools/acp/list-agents.test.ts +3 -1
  781. package/src/tools/acp/spawn.test.ts +146 -11
  782. package/src/tools/acp/spawn.ts +21 -3
  783. package/src/tools/acp/status.test.ts +349 -0
  784. package/src/tools/acp/status.ts +112 -9
  785. package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
  786. package/src/tools/browser/browser-execution.ts +45 -14
  787. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  788. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
  789. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  790. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
  791. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  792. package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
  793. package/src/tools/browser/network-recorder.ts +2 -1
  794. package/src/tools/capability-offer.test.ts +22 -2
  795. package/src/tools/capability-offer.ts +16 -1
  796. package/src/tools/computer-use/definitions.ts +123 -5
  797. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
  798. package/src/tools/credentials/metadata-store.ts +100 -0
  799. package/src/tools/filesystem/search.ts +2 -1
  800. package/src/tools/host-filesystem/edit.test.ts +2 -1
  801. package/src/tools/host-filesystem/edit.ts +2 -2
  802. package/src/tools/host-filesystem/read.test.ts +2 -1
  803. package/src/tools/host-filesystem/read.ts +2 -2
  804. package/src/tools/host-filesystem/transfer.test.ts +2 -1
  805. package/src/tools/host-filesystem/transfer.ts +2 -2
  806. package/src/tools/host-filesystem/write.test.ts +2 -1
  807. package/src/tools/host-filesystem/write.ts +2 -2
  808. package/src/tools/host-terminal/host-shell.test.ts +16 -0
  809. package/src/tools/host-terminal/host-shell.ts +16 -26
  810. package/src/tools/network/web-search.ts +6 -1
  811. package/src/tools/policy-context.ts +6 -7
  812. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  813. package/src/tools/shared/shell-output.test.ts +10 -0
  814. package/src/tools/shared/shell-output.ts +14 -2
  815. package/src/tools/skills/find-similar-skills.test.ts +214 -2
  816. package/src/tools/skills/find-similar-skills.ts +62 -9
  817. package/src/tools/skills/resolve-execute-invocation.ts +24 -0
  818. package/src/tools/skills/sandbox-runner.ts +13 -2
  819. package/src/tools/skills/scaffold-managed.ts +78 -16
  820. package/src/tools/subagent/spawn.ts +5 -8
  821. package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
  822. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +43 -0
  823. package/src/tools/terminal/safe-env.ts +35 -24
  824. package/src/tools/terminal/sanitized-bash.ts +118 -0
  825. package/src/tools/terminal/shell-launch.test.ts +162 -0
  826. package/src/tools/terminal/shell.test.ts +29 -0
  827. package/src/tools/terminal/shell.ts +13 -7
  828. package/src/tools/ui-surface/channel-variants.ts +10 -59
  829. package/src/tools/watch/watch-retro-report.ts +0 -8
  830. package/src/types/host-capabilities.ts +5 -3
  831. package/src/util/__tests__/language-subtag.test.ts +23 -1
  832. package/src/util/abort-reasons.ts +12 -1
  833. package/src/util/ca-bundle.ts +13 -0
  834. package/src/util/host-process.test.ts +17 -1
  835. package/src/util/host-process.ts +24 -0
  836. package/src/util/language-subtag.ts +16 -0
  837. package/src/util/platform.ts +8 -0
  838. package/src/util/plugin-manifest.ts +203 -0
  839. package/src/util/unicode.ts +44 -1
  840. package/src/watch/watch-retro.ts +6 -8
  841. package/src/watch/watch-timeline.ts +2 -2
  842. package/src/workflows/run-manager.ts +2 -1
  843. package/src/workspace/git-porcelain.ts +66 -0
  844. package/src/workspace/git-service.ts +6 -55
  845. package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
  846. package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
  847. package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
  848. package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
  849. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
  850. package/src/workspace/migrations/registry.ts +6 -0
  851. package/src/__tests__/app-source-watcher.test.ts +0 -218
  852. package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
  853. package/src/__tests__/speaker-identification.test.ts +0 -56
  854. package/src/calls/speaker-identification.ts +0 -233
  855. package/src/daemon/app-source-watcher.ts +0 -218
  856. package/src/messaging/providers/whatsapp/client.ts +0 -29
  857. package/src/runtime/routes/debug-bash-routes.ts +0 -173
  858. package/src/util/fs-watcher-error.ts +0 -36
  859. /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
@@ -0,0 +1,423 @@
1
+ /**
2
+ * Progress narration cadence for one voice turn, shared by the live-voice
3
+ * session and the phone call controller.
4
+ *
5
+ * A voice turn that runs tools is silent while they run. The cadence speaks
6
+ * short audio-only progress updates into that dead air: a generated
7
+ * sentence about what the turn is doing (see `progress-narration.ts`), or a
8
+ * static fallback phrase when generation yields nothing. It follows the
9
+ * turn's tool activity rather than the clock: an "ops" burst, a long
10
+ * operation completing, or a full interval of audible silence with news
11
+ * since the last update, with `maxSilenceMs` as the heartbeat ceiling.
12
+ *
13
+ * The cadence owns the per-turn state (the tool-activity log, epochs,
14
+ * spacing, the self-re-arming idle timer). Everything transport-specific
15
+ * comes in through {@link ProgressCadenceHost}: whether the turn's audio is
16
+ * idle, when it last reached the listener, and how to speak a phrase.
17
+ */
18
+
19
+ import type { VoiceProgressConfig } from "../config/schemas/voice.js";
20
+ import { fixedPhraseLanguage } from "../util/language-subtag.js";
21
+ import { getLogger } from "../util/logger.js";
22
+ import type {
23
+ VoiceProgressNarrator,
24
+ VoiceProgressTextInput,
25
+ } from "./progress-narration.js";
26
+ import {
27
+ pickProgressPhrase,
28
+ PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
29
+ } from "./progress-phrases.js";
30
+
31
+ const log = getLogger("voice-progress-cadence");
32
+
33
+ /**
34
+ * One tool operation observed on a turn, fed by the bridge's structured tool
35
+ * callbacks. `completedAtMs`/`isError`/`resultPreview` land with tool_result.
36
+ */
37
+ export interface TurnProgressOp {
38
+ toolName: string;
39
+ toolUseId?: string;
40
+ startedAtMs: number;
41
+ completedAtMs?: number;
42
+ isError?: boolean;
43
+ resultPreview?: string;
44
+ }
45
+
46
+ /** What prompted a narration attempt. */
47
+ export type ProgressNarrationTrigger = "ops" | "idle" | "op_complete";
48
+
49
+ /** A tool completion as the bridge reports it. */
50
+ export interface ProgressToolResult {
51
+ toolName: string;
52
+ toolUseId?: string;
53
+ isError?: boolean;
54
+ resultPreview?: string;
55
+ }
56
+
57
+ /**
58
+ * Newest incomplete op, optionally restricted to a tool name (the
59
+ * tool_result fallback match when no toolUseId correlates).
60
+ */
61
+ export function findLastIncompleteOp(
62
+ ops: readonly TurnProgressOp[],
63
+ toolName?: string,
64
+ ): TurnProgressOp | undefined {
65
+ for (let i = ops.length - 1; i >= 0; i -= 1) {
66
+ const op = ops[i];
67
+ if (
68
+ op.completedAtMs === undefined &&
69
+ (toolName === undefined || op.toolName === toolName)
70
+ ) {
71
+ return op;
72
+ }
73
+ }
74
+ return undefined;
75
+ }
76
+
77
+ /**
78
+ * The transport-specific half of progress narration, supplied by the driver
79
+ * that owns the turn's audio.
80
+ */
81
+ export interface ProgressCadenceHost {
82
+ /** Identifies the turn in logs. */
83
+ turnId: string;
84
+ /** Dispatch timestamp of the turn (`Date.now()` at launch). */
85
+ launchedAtMs: number;
86
+ /** Aborts an in-flight narration generation with the turn. */
87
+ signal: AbortSignal;
88
+ /**
89
+ * Whether the turn may still be narrated: it is the live turn, it has not
90
+ * completed, and it is not waiting on a decision (a turn waiting on an
91
+ * approval says so once and is quiet after that).
92
+ */
93
+ canNarrate(): boolean;
94
+ /**
95
+ * No assistant audio is buffered, queued, in flight, or (estimatedly)
96
+ * still playing to the listener.
97
+ */
98
+ isAudioIdle(): boolean;
99
+ /**
100
+ * Wall-clock estimate of when audio already handed to the transport
101
+ * finishes reaching the listener, 0 when nothing is playing. Together with
102
+ * the cadence's own last-emitted stamp this anchors the dead-air countdown
103
+ * to when the listener last heard something.
104
+ */
105
+ playbackTailUntilMs(): number;
106
+ /** The caller's words this turn answers, for the narrator. */
107
+ transcriptSoFar(): string;
108
+ /** The turn's spoken language, when known. */
109
+ language(): string | undefined;
110
+ /**
111
+ * Bumped on every assistant text delta. A narration generated while the
112
+ * model was already speaking is stale and is dropped.
113
+ */
114
+ deltaEpoch(): number;
115
+ /**
116
+ * Speak one sanitized phrase, audio-only. `language` is the TTS hint the
117
+ * phrase must carry ("en" for a static fallback the localized table lacks
118
+ * in the turn's language); undefined rides the turn's language. Returns
119
+ * false when nothing was spoken.
120
+ */
121
+ speak(phrase: string, language: string | undefined): boolean;
122
+ /** Called after a narration was actually enqueued (metrics). */
123
+ onNarrationSpoken?(): void;
124
+ }
125
+
126
+ export interface ProgressCadenceOptions {
127
+ config: VoiceProgressConfig;
128
+ /** Null disables narration for the turn (no TTS, or narration off). */
129
+ narrator: VoiceProgressNarrator | null;
130
+ host: ProgressCadenceHost;
131
+ /**
132
+ * Rotates the static fallback phrase across turns so a session with
133
+ * repeated dead air does not repeat one sentence.
134
+ */
135
+ nextFallbackPhraseIndex: () => number;
136
+ }
137
+
138
+ export interface ProgressCadence {
139
+ /** Tool operations this turn, in start order. */
140
+ readonly ops: readonly TurnProgressOp[];
141
+ /** Narrations actually spoken this turn. */
142
+ readonly updatesSpoken: number;
143
+ /** A tool started: counts toward the "ops" trigger and is news for idle. */
144
+ toolStarted(toolName: string, toolUseId?: string): void;
145
+ /**
146
+ * A tool finished. A long operation completing narrates on its own beat;
147
+ * a quick one only counts as news.
148
+ */
149
+ toolFinished(event: ProgressToolResult): void;
150
+ /** The turn's audio last finished emitting now: restarts the dead-air clock. */
151
+ noteAudioSettled(): void;
152
+ /**
153
+ * Something else held the floor now (a spoken approval line, a bridge):
154
+ * narration keeps `minGapMs` from it.
155
+ */
156
+ noteFloorHolder(): void;
157
+ /**
158
+ * Arm (or re-arm) the dead-air timer. A no-op when narration is off for
159
+ * the turn.
160
+ */
161
+ arm(): void;
162
+ /** Stop the dead-air timer; the turn ended, was cancelled, or handed off. */
163
+ clear(): void;
164
+ }
165
+
166
+ export function createProgressCadence(
167
+ options: ProgressCadenceOptions,
168
+ ): ProgressCadence {
169
+ const { config, narrator, host, nextFallbackPhraseIndex } = options;
170
+ const ops: TurnProgressOp[] = [];
171
+ // Ops accumulated toward the next ops-triggered narration. Counted once
172
+ // per op, on start (not completion), so a burst of slow tools still trips
173
+ // the threshold while they run.
174
+ let opsSinceNarration = 0;
175
+ // Bumped by every observable change to the turn's tool activity. The idle
176
+ // trigger compares it against `narratedEpoch` so a tick with nothing new
177
+ // to report stays silent.
178
+ let stateEpoch = 0;
179
+ let narratedEpoch = 0;
180
+ let updatesSpoken = 0;
181
+ // When the last spoken floor-holder enqueued; gates `minGapMs`.
182
+ let lastFloorHolderAtMs: number | null = null;
183
+ // When the turn's audio last finished emitting (turn launch until anything
184
+ // speaks). Idle time is measured from when the listener last heard
185
+ // something, not from when the turn started.
186
+ let lastAudibleAtMs = Date.now();
187
+ let idleTimer: ReturnType<typeof setTimeout> | null = null;
188
+ // A narration generation is awaiting the narrator; serializes narrations.
189
+ let narrationInFlight = false;
190
+
191
+ const enabled = config.enabled && narrator !== null;
192
+
193
+ // When the turn's current audible silence began: the latest of the last
194
+ // emitted segment, the estimated playback end, and the last floor holder.
195
+ const silenceSinceMs = (): number =>
196
+ Math.max(
197
+ lastAudibleAtMs,
198
+ host.playbackTailUntilMs(),
199
+ lastFloorHolderAtMs ?? 0,
200
+ );
201
+
202
+ const canNarrate = (): boolean => host.canNarrate() && host.isAudioIdle();
203
+
204
+ const withinMinGap = (): boolean =>
205
+ lastFloorHolderAtMs !== null &&
206
+ Date.now() - lastFloorHolderAtMs < config.minGapMs;
207
+
208
+ // The idle tick has something worth saying when the turn's tool activity
209
+ // has moved since the last narration described it, or when the silence has
210
+ // run past `maxSilenceMs`, the heartbeat ceiling that proves the assistant
211
+ // is still alive on a turn with no observable activity at all.
212
+ const idleHasSomethingToSay = (): boolean => {
213
+ if (stateEpoch !== narratedEpoch) {
214
+ return true;
215
+ }
216
+ const silentForMs = Date.now() - silenceSinceMs();
217
+ if (silentForMs >= config.maxSilenceMs) {
218
+ return true;
219
+ }
220
+ log.debug(
221
+ { turnId: host.turnId, silentForMs },
222
+ "Voice progress narration held: nothing new since the last update",
223
+ );
224
+ return false;
225
+ };
226
+
227
+ const clearTimer = (): void => {
228
+ if (idleTimer !== null) {
229
+ clearTimeout(idleTimer);
230
+ idleTimer = null;
231
+ }
232
+ };
233
+
234
+ // Arms the dead-air timer. The countdown measures audible silence, not
235
+ // time since launch, so it covers mid-turn silences for the whole turn. On
236
+ // expiry with audio still pending, or with the silence not yet a full
237
+ // interval old, it re-arms for the remainder; only a full interval of
238
+ // audible silence reaches the gatekeeper. The interval is a polling
239
+ // cadence, not a speaking cadence: most ticks find nothing new and stay
240
+ // quiet.
241
+ const armTimer = (delayMs?: number): void => {
242
+ clearTimer();
243
+ idleTimer = setTimeout(() => {
244
+ idleTimer = null;
245
+ if (!host.canNarrate()) {
246
+ return;
247
+ }
248
+ if (!host.isAudioIdle()) {
249
+ armTimer();
250
+ return;
251
+ }
252
+ const remaining = silenceSinceMs() + config.idleIntervalMs - Date.now();
253
+ if (remaining > 0) {
254
+ armTimer(remaining);
255
+ return;
256
+ }
257
+ maybeNarrate("idle");
258
+ armTimer();
259
+ }, delayMs ?? config.idleIntervalMs);
260
+ };
261
+
262
+ // Generate and speak one audio-only progress narration. On a null result,
263
+ // idle narration falls back to a static phrase while activity-triggered
264
+ // narration stays silent.
265
+ const speakUpdate = async (
266
+ activeNarrator: VoiceProgressNarrator,
267
+ trigger: ProgressNarrationTrigger,
268
+ ): Promise<void> => {
269
+ narrationInFlight = true;
270
+ const deltaEpochAtLaunch = host.deltaEpoch();
271
+ // The activity this update describes. Tool events that land
272
+ // mid-generation are news the generated text cannot carry, so they must
273
+ // leave the idle trigger armed rather than count as already narrated.
274
+ const stateEpochAtLaunch = stateEpoch;
275
+ try {
276
+ const now = Date.now();
277
+ const currentOp = findLastIncompleteOp(ops);
278
+ const language = host.language();
279
+ const input: VoiceProgressTextInput = {
280
+ transcriptSoFar: host.transcriptSoFar(),
281
+ completedOps: ops
282
+ .filter(
283
+ (op): op is TurnProgressOp & { completedAtMs: number } =>
284
+ op.completedAtMs !== undefined,
285
+ )
286
+ // `ops` is in start order; the narrator wants completion order,
287
+ // which differs when parallel tools finish out of order.
288
+ .sort((a, b) => a.completedAtMs - b.completedAtMs)
289
+ .map((op) => ({
290
+ toolName: op.toolName,
291
+ ...(op.isError !== undefined ? { isError: op.isError } : {}),
292
+ ...(op.resultPreview !== undefined
293
+ ? { resultPreview: op.resultPreview }
294
+ : {}),
295
+ })),
296
+ currentOp: currentOp
297
+ ? {
298
+ toolName: currentOp.toolName,
299
+ elapsedMs: now - currentOp.startedAtMs,
300
+ }
301
+ : null,
302
+ turnElapsedMs: now - host.launchedAtMs,
303
+ updateIndex: updatesSpoken + 1,
304
+ ...(language !== undefined ? { languageHint: language } : {}),
305
+ };
306
+ const generated = await activeNarrator
307
+ .generateProgressText(input, host.signal)
308
+ // The narrator contract never rejects. Keep test stubs fail-soft too.
309
+ .catch(() => null);
310
+ // Re-check liveness and staleness after the provider call.
311
+ if (
312
+ !canNarrate() ||
313
+ host.deltaEpoch() !== deltaEpochAtLaunch ||
314
+ withinMinGap()
315
+ ) {
316
+ return;
317
+ }
318
+ let raw = generated;
319
+ // Generated text is in the turn's language; only the static fallback
320
+ // comes from a localized table and may need the "en" override.
321
+ let fillerLanguage: string | undefined;
322
+ if (raw === null) {
323
+ if (trigger !== "idle") {
324
+ return;
325
+ }
326
+ raw = pickProgressPhrase(nextFallbackPhraseIndex(), language);
327
+ fillerLanguage = fixedPhraseLanguage(
328
+ PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
329
+ language,
330
+ );
331
+ }
332
+ if (!host.speak(raw, fillerLanguage)) {
333
+ return;
334
+ }
335
+ lastFloorHolderAtMs = Date.now();
336
+ opsSinceNarration = 0;
337
+ narratedEpoch = stateEpochAtLaunch;
338
+ updatesSpoken += 1;
339
+ host.onNarrationSpoken?.();
340
+ // Restart the dead-air countdown from this narration.
341
+ armTimer();
342
+ } finally {
343
+ narrationInFlight = false;
344
+ }
345
+ };
346
+
347
+ // Gatekeeper. Speaks only while the turn is audibly silent, with one
348
+ // generation at a time and the configured spacing.
349
+ const maybeNarrate = (trigger: ProgressNarrationTrigger): void => {
350
+ if (
351
+ !enabled ||
352
+ narrator === null ||
353
+ !canNarrate() ||
354
+ narrationInFlight ||
355
+ withinMinGap() ||
356
+ (trigger === "ops" && opsSinceNarration < config.opsThreshold) ||
357
+ (trigger === "idle" && !idleHasSomethingToSay())
358
+ ) {
359
+ return;
360
+ }
361
+ void speakUpdate(narrator, trigger);
362
+ };
363
+
364
+ return {
365
+ get ops() {
366
+ return ops;
367
+ },
368
+ get updatesSpoken() {
369
+ return updatesSpoken;
370
+ },
371
+ toolStarted(toolName, toolUseId) {
372
+ ops.push({
373
+ toolName,
374
+ ...(toolUseId !== undefined ? { toolUseId } : {}),
375
+ startedAtMs: Date.now(),
376
+ });
377
+ opsSinceNarration += 1;
378
+ stateEpoch += 1;
379
+ maybeNarrate("ops");
380
+ },
381
+ toolFinished(event) {
382
+ // Match by toolUseId when present; otherwise the last-started
383
+ // incomplete op with the same name (parallel same-name ops resolve
384
+ // newest-first).
385
+ const op =
386
+ (event.toolUseId !== undefined
387
+ ? ops.find((o) => o.toolUseId === event.toolUseId)
388
+ : undefined) ?? findLastIncompleteOp(ops, event.toolName);
389
+ // A long-running op finishing is the beat the listener has been
390
+ // waiting through: it narrates immediately rather than waiting for
391
+ // `opsThreshold` more ops, which on a one-slow-tool turn never arrive.
392
+ // Short ops stay on the ops trigger.
393
+ let trigger: ProgressNarrationTrigger = "ops";
394
+ if (op) {
395
+ op.completedAtMs = Date.now();
396
+ if (event.isError !== undefined) {
397
+ op.isError = event.isError;
398
+ }
399
+ if (event.resultPreview !== undefined) {
400
+ op.resultPreview = event.resultPreview;
401
+ }
402
+ if (op.completedAtMs - op.startedAtMs >= config.longOpMs) {
403
+ trigger = "op_complete";
404
+ }
405
+ }
406
+ stateEpoch += 1;
407
+ maybeNarrate(trigger);
408
+ },
409
+ noteAudioSettled() {
410
+ lastAudibleAtMs = Date.now();
411
+ },
412
+ noteFloorHolder() {
413
+ lastFloorHolderAtMs = Date.now();
414
+ },
415
+ arm() {
416
+ if (!enabled) {
417
+ return;
418
+ }
419
+ armTimer();
420
+ },
421
+ clear: clearTimer,
422
+ };
423
+ }