@vellumai/assistant 0.11.1 → 0.11.2-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (318) hide show
  1. package/Dockerfile +1 -3
  2. package/README.md +1 -1
  3. package/eslint.config.mjs +28 -8
  4. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  5. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
  6. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
  7. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  8. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
  9. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
  10. package/node_modules/@vellumai/service-contracts/package.json +1 -0
  11. package/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
  12. package/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
  13. package/node_modules/@vellumai/slack-text/src/index.test.ts +48 -0
  14. package/node_modules/@vellumai/slack-text/src/index.ts +25 -6
  15. package/openapi.yaml +292 -5
  16. package/package.json +1 -1
  17. package/src/__tests__/agent-loop-resume-interrupted.test.ts +223 -0
  18. package/src/__tests__/byok-default-profile-ensure.test.ts +43 -0
  19. package/src/__tests__/cli-logger-boundary-guard.test.ts +87 -0
  20. package/src/__tests__/compaction-events.test.ts +139 -2
  21. package/src/__tests__/config-loader-backfill.test.ts +35 -3
  22. package/src/__tests__/config-schema.test.ts +1 -0
  23. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +30 -0
  24. package/src/__tests__/conversation-agent-loop.test.ts +180 -7
  25. package/src/__tests__/conversation-error.test.ts +211 -4
  26. package/src/__tests__/conversation-options-turn-scoped-transport.test.ts +64 -0
  27. package/src/__tests__/conversation-queue.test.ts +126 -0
  28. package/src/__tests__/conversation-retry-route.test.ts +46 -7
  29. package/src/__tests__/conversation-slash.test.ts +4 -2
  30. package/src/__tests__/conversation-summarize-route.test.ts +7 -1
  31. package/src/__tests__/conversation-summarize-up-to.test.ts +71 -5
  32. package/src/__tests__/document-create-dedupe.test.ts +2 -1
  33. package/src/__tests__/document-find-replace.test.ts +2 -1
  34. package/src/__tests__/document-tool-security.test.ts +2 -1
  35. package/src/__tests__/document-update-default-surface.test.ts +2 -1
  36. package/src/__tests__/document-workspace-file.test.ts +467 -0
  37. package/src/__tests__/emit-signal-routing-intent.test.ts +290 -49
  38. package/src/__tests__/guardian-card-withdrawal.test.ts +91 -3
  39. package/src/__tests__/http-user-message-parity.test.ts +66 -0
  40. package/src/__tests__/list-messages-attachments.test.ts +87 -0
  41. package/src/__tests__/list-messages-provider-error.test.ts +143 -0
  42. package/src/__tests__/list-messages-system-card.test.ts +100 -0
  43. package/src/__tests__/llm-resolver.test.ts +19 -9
  44. package/src/__tests__/managed-profile-guard.test.ts +23 -0
  45. package/src/__tests__/notification-platform-adapter.test.ts +130 -2
  46. package/src/__tests__/notification-telegram-adapter.test.ts +6 -0
  47. package/src/__tests__/notification-vellum-adapter.test.ts +45 -0
  48. package/src/__tests__/plugin-api-model-profiles.test.ts +10 -1
  49. package/src/__tests__/plugin-import-boundary-guard.test.ts +3 -0
  50. package/src/__tests__/provider-error-scenarios.test.ts +140 -0
  51. package/src/__tests__/provider-send-message-override-profile.test.ts +95 -0
  52. package/src/__tests__/run-conversation-turn-persistence.test.ts +66 -3
  53. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +209 -0
  54. package/src/__tests__/skills.test.ts +27 -0
  55. package/src/__tests__/slack-channels-routes.test.ts +0 -2
  56. package/src/__tests__/slack-share-routes.test.ts +0 -3
  57. package/src/__tests__/slack-users-routes.test.ts +0 -2
  58. package/src/__tests__/subagent-tools.test.ts +11 -0
  59. package/src/__tests__/tool-preview-lifecycle.test.ts +58 -0
  60. package/src/__tests__/tool-result-spool.test.ts +5 -4
  61. package/src/__tests__/turn-boundary-resolution.test.ts +53 -0
  62. package/src/__tests__/turn-events-store.test.ts +26 -0
  63. package/src/__tests__/ui-visual-surface.test.ts +695 -0
  64. package/src/__tests__/unified-turn-context-visible-app.test.ts +99 -0
  65. package/src/__tests__/visible-app-context.test.ts +189 -0
  66. package/src/__tests__/workspace-git-service.test.ts +173 -33
  67. package/src/__tests__/workspace-migration-137-repair-retired-fireworks-minimax-model-id.test.ts +157 -0
  68. package/src/__tests__/workspace-migration-138-backfill-home-feed-titles.test.ts +373 -0
  69. package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +137 -0
  70. package/src/agent/loop.ts +36 -7
  71. package/src/api/events/context-window-usage.ts +31 -0
  72. package/src/api/events/notification-intent.ts +8 -0
  73. package/src/api/events/ui-surface-pending.ts +35 -0
  74. package/src/api/index.ts +14 -0
  75. package/src/api/responses/conversation-message.ts +25 -4
  76. package/src/api/surfaces.ts +90 -1
  77. package/src/approvals/guardian-card-withdrawal.ts +66 -31
  78. package/src/approvals/guardian-decision-primitive.ts +4 -0
  79. package/src/calls/__tests__/call-setup-router.test.ts +156 -26
  80. package/src/calls/__tests__/voice-session-bridge.test.ts +201 -12
  81. package/src/calls/call-setup-router.ts +80 -36
  82. package/src/calls/voice-session-bridge.ts +173 -26
  83. package/src/cli/commands/inference.help.ts +3 -3
  84. package/src/cli/commands/notifications.help.ts +3 -2
  85. package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +42 -128
  86. package/src/cli/commands/platform/__tests__/credits.test.ts +9 -78
  87. package/src/cli/commands/platform/__tests__/helpers.ts +90 -0
  88. package/src/cli/commands/platform/__tests__/invoices.test.ts +238 -0
  89. package/src/cli/commands/platform/__tests__/plans.test.ts +9 -88
  90. package/src/cli/commands/platform/__tests__/status.test.ts +12 -87
  91. package/src/cli/commands/platform/__tests__/subscription.test.ts +9 -86
  92. package/src/cli/commands/platform/index.help.ts +92 -0
  93. package/src/cli/commands/platform/index.ts +7 -0
  94. package/src/cli/commands/platform/invoices.ts +132 -0
  95. package/src/cli/commands/usage.help.ts +1 -1
  96. package/src/cli/lib/list-installed-plugins.ts +2 -1
  97. package/src/config/__tests__/default-profile-catalog.test.ts +7 -10
  98. package/src/config/__tests__/deployment-context-defaults.test.ts +27 -5
  99. package/src/config/assistant-feature-flags.ts +7 -2
  100. package/src/config/bundled-skills/app-builder/SKILL.md +3 -2
  101. package/src/config/bundled-skills/subagent/SKILL.md +3 -1
  102. package/src/config/bundled-skills/subagent/TOOLS.json +3 -3
  103. package/src/config/bundled-skills/visualize/SKILL.md +163 -0
  104. package/src/config/call-site-defaults.ts +3 -2
  105. package/src/config/default-profile-catalog.ts +75 -70
  106. package/src/config/default-profile-names.ts +13 -28
  107. package/src/config/env-registry.ts +1 -0
  108. package/src/config/feature-flag-registry.json +16 -0
  109. package/src/config/llm-resolver.ts +21 -5
  110. package/src/config/loader.ts +18 -11
  111. package/src/config/schemas/llm.ts +1 -9
  112. package/src/config/schemas/memory-retrospective.ts +9 -0
  113. package/src/config/schemas/monitoring.ts +28 -2
  114. package/src/config/schemas/workspace-git.ts +15 -0
  115. package/src/config/seed-inference-profiles.ts +19 -5
  116. package/src/context/post-turn-tool-result-truncation.ts +2 -2
  117. package/src/conversations/__tests__/message-consolidation.test.ts +54 -0
  118. package/src/conversations/message-consolidation.ts +17 -15
  119. package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +182 -0
  120. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +181 -0
  121. package/src/daemon/conversation-agent-loop-handlers.ts +90 -7
  122. package/src/daemon/conversation-agent-loop.ts +107 -22
  123. package/src/daemon/conversation-error.ts +169 -48
  124. package/src/daemon/conversation-messaging.ts +87 -4
  125. package/src/daemon/conversation-process.ts +57 -45
  126. package/src/daemon/conversation-runtime-assembly.ts +57 -0
  127. package/src/daemon/conversation-store.ts +36 -1
  128. package/src/daemon/conversation-surfaces.ts +68 -6
  129. package/src/daemon/conversation-turn-finalize.ts +76 -18
  130. package/src/daemon/conversation.ts +122 -27
  131. package/src/daemon/lifecycle.ts +9 -0
  132. package/src/daemon/message-types/conversations.ts +8 -0
  133. package/src/daemon/message-types/surfaces.ts +3 -0
  134. package/src/documents/document-store.ts +247 -10
  135. package/src/home/__tests__/feed-types.test.ts +43 -8
  136. package/src/home/__tests__/feed-writer.test.ts +59 -0
  137. package/src/home/feed-types.ts +34 -7
  138. package/src/home/feed-writer.ts +15 -6
  139. package/src/live-voice/__tests__/activity-label.test.ts +95 -0
  140. package/src/live-voice/__tests__/live-activity-reporter.test.ts +86 -5
  141. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +310 -25
  142. package/src/live-voice/__tests__/live-voice-events.test.ts +4 -4
  143. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +8 -4
  144. package/src/live-voice/__tests__/live-voice-vad.test.ts +9 -1
  145. package/src/live-voice/activity-label.ts +169 -0
  146. package/src/live-voice/live-activity-reporter.ts +35 -6
  147. package/src/live-voice/live-voice-session.ts +282 -41
  148. package/src/live-voice/protocol.ts +35 -0
  149. package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +84 -2
  150. package/src/messaging/providers/slack/adapter.ts +86 -19
  151. package/src/messaging/providers/slack/api.test.ts +85 -1
  152. package/src/messaging/providers/slack/api.ts +121 -288
  153. package/src/messaging/providers/slack/client.ts +20 -251
  154. package/src/messaging/providers/slack/send.test.ts +4 -9
  155. package/src/messaging/providers/slack/send.ts +1 -1
  156. package/src/messaging/providers/slack/types.ts +5 -0
  157. package/src/messaging/providers/slack/web-api-transport.test.ts +181 -0
  158. package/src/messaging/providers/slack/web-api-transport.ts +367 -0
  159. package/src/messaging/providers/slack/withdraw.ts +6 -14
  160. package/src/messaging/providers/telegram-bot/send.test.ts +31 -1
  161. package/src/messaging/providers/telegram-bot/send.ts +23 -2
  162. package/src/messaging/providers/telegram-bot/withdraw.test.ts +152 -0
  163. package/src/messaging/providers/telegram-bot/withdraw.ts +166 -0
  164. package/src/monitoring/__tests__/db-integrity-sample.test.ts +18 -4
  165. package/src/monitoring/__tests__/file-descriptors.test.ts +144 -0
  166. package/src/monitoring/file-descriptors.ts +262 -0
  167. package/src/monitoring/process-memory.ts +4 -11
  168. package/src/monitoring/resource-sampler.ts +5 -0
  169. package/src/monitoring/worker.ts +12 -0
  170. package/src/notifications/__tests__/assistant-reply-producer.test.ts +740 -0
  171. package/src/notifications/__tests__/broadcaster.test.ts +347 -7
  172. package/src/notifications/__tests__/copy-composer.test.ts +53 -3
  173. package/src/notifications/__tests__/decision-engine.test.ts +349 -29
  174. package/src/notifications/__tests__/deterministic-checks.test.ts +21 -0
  175. package/src/notifications/__tests__/edit-notification.test.ts +330 -0
  176. package/src/notifications/__tests__/guardian-delivery-recorder.test.ts +71 -0
  177. package/src/notifications/__tests__/home-feed-side-effect.test.ts +170 -12
  178. package/src/notifications/adapters/macos.ts +3 -0
  179. package/src/notifications/adapters/platform.ts +90 -14
  180. package/src/notifications/adapters/telegram.ts +6 -4
  181. package/src/notifications/assistant-reply-producer.ts +219 -0
  182. package/src/notifications/broadcaster.ts +523 -317
  183. package/src/notifications/copy-composer.ts +27 -6
  184. package/src/notifications/decision-engine.ts +154 -77
  185. package/src/notifications/deterministic-checks.ts +6 -7
  186. package/src/notifications/edit-notification.ts +8 -4
  187. package/src/notifications/emit-signal.ts +49 -16
  188. package/src/notifications/guardian-delivery-recorder.ts +8 -2
  189. package/src/notifications/home-feed-side-effect.ts +43 -6
  190. package/src/notifications/notification-utils.ts +39 -4
  191. package/src/notifications/signal.ts +10 -0
  192. package/src/notifications/types.ts +17 -0
  193. package/src/persistence/bookmark-crud.ts +18 -9
  194. package/src/persistence/conversation-attention-store.ts +26 -0
  195. package/src/persistence/conversation-crud.ts +79 -32
  196. package/src/persistence/conversation-queries.ts +30 -15
  197. package/src/persistence/conversation-title-service.ts +5 -170
  198. package/src/persistence/conversation-types.ts +192 -1
  199. package/src/persistence/db-init.ts +14 -1
  200. package/src/persistence/db-maintenance.ts +5 -4
  201. package/src/persistence/embeddings/__tests__/plugin-index-qdrant-init.test.ts +219 -0
  202. package/src/persistence/embeddings/__tests__/plugin-index.test.ts +22 -0
  203. package/src/persistence/embeddings/__tests__/worker-script-version.test.ts +76 -0
  204. package/src/persistence/embeddings/embedding-local.ts +2 -1
  205. package/src/persistence/embeddings/embedding-runtime-manager.ts +29 -5
  206. package/src/persistence/embeddings/plugin-index.ts +84 -12
  207. package/src/persistence/migrations/360-add-document-workspace-path.test.ts +110 -0
  208. package/src/persistence/migrations/360-add-document-workspace-path.ts +39 -0
  209. package/src/persistence/planner-statistics.ts +14 -0
  210. package/src/persistence/schema/documents.ts +26 -11
  211. package/src/persistence/steps.ts +2 -0
  212. package/src/platform/client.test.ts +172 -2
  213. package/src/platform/client.ts +178 -55
  214. package/src/plugin-api/conversation-turn.ts +13 -0
  215. package/src/plugin-api/model-profiles.test.ts +6 -2
  216. package/src/plugins/defaults/memory/__tests__/bookmark-crud.test.ts +34 -0
  217. package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +22 -0
  218. package/src/plugins/defaults/memory/__tests__/jobs-store-enqueue-gate.test.ts +11 -0
  219. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +199 -0
  220. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +98 -1
  221. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +105 -1
  222. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +24 -2
  223. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +524 -0
  224. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1 -1
  225. package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +9 -4
  226. package/src/plugins/defaults/memory/host-utils.ts +5 -0
  227. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +105 -1
  228. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +64 -5
  229. package/src/plugins/defaults/memory/memory-retrospective-job.ts +36 -1
  230. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +12 -3
  231. package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +1 -0
  232. package/src/plugins/defaults/memory/substrate/__tests__/skill-store.test.ts +195 -1
  233. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +1 -1
  234. package/src/plugins/defaults/memory/substrate/skill-store.ts +167 -29
  235. package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +131 -0
  236. package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +1 -0
  237. package/src/plugins/defaults/memory/v2/activation-log-store.ts +4 -0
  238. package/src/plugins/defaults/memory/v2/injection.ts +48 -3
  239. package/src/plugins/defaults/memory/v2/rerank-local.ts +6 -2
  240. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +1 -1
  241. package/src/plugins/defaults/platform-hosted/routes/reengage.ts +1 -1
  242. package/src/plugins/defaults/turn-context/injectors.ts +1 -0
  243. package/src/plugins/defaults/turn-context/unified-turn-context.ts +26 -0
  244. package/src/plugins/types.ts +17 -0
  245. package/src/prompts/templates/system-sections.ts +2 -2
  246. package/src/providers/__tests__/dispatch-connection-routing.test.ts +43 -2
  247. package/src/providers/__tests__/vellum-mismatch-routing.test.ts +8 -0
  248. package/src/providers/call-site-routing.ts +77 -14
  249. package/src/providers/connection-resolution.ts +42 -2
  250. package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +127 -1
  251. package/src/providers/inference/adapter-factory.ts +70 -12
  252. package/src/providers/model-catalog.ts +0 -11
  253. package/src/providers/model-intents.ts +11 -0
  254. package/src/providers/registry.ts +8 -0
  255. package/src/providers/retry.ts +85 -17
  256. package/src/providers/types.ts +8 -1
  257. package/src/runtime/__tests__/agent-wake.test.ts +181 -0
  258. package/src/runtime/agent-wake.ts +132 -9
  259. package/src/runtime/channel-approval-types.ts +25 -0
  260. package/src/runtime/routes/__tests__/connection-routes-vs-cli-parity.test.ts +2 -2
  261. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +11 -0
  262. package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +121 -9
  263. package/src/runtime/routes/__tests__/platform-invoice-routes.test.ts +392 -0
  264. package/src/runtime/routes/__tests__/slack-channel-routes.test.ts +7 -0
  265. package/src/runtime/routes/__tests__/workspace-commit-routes.test.ts +63 -0
  266. package/src/runtime/routes/consolidation-routes.ts +1 -1
  267. package/src/runtime/routes/conversation-management-routes.ts +18 -6
  268. package/src/runtime/routes/conversation-query-routes.ts +14 -23
  269. package/src/runtime/routes/conversation-routes.ts +113 -8
  270. package/src/runtime/routes/credential-routes.ts +1 -5
  271. package/src/runtime/routes/documents-routes.ts +207 -12
  272. package/src/runtime/routes/identity-routes.ts +3 -93
  273. package/src/runtime/routes/inbound-stages/admission-policy.test.ts +31 -1
  274. package/src/runtime/routes/inbound-stages/admission-policy.ts +18 -0
  275. package/src/runtime/routes/inference-provider-connection-routes.ts +52 -3
  276. package/src/runtime/routes/notification-routes.ts +4 -1
  277. package/src/runtime/routes/platform-routes.ts +238 -3
  278. package/src/runtime/routes/playground/guard.ts +1 -2
  279. package/src/runtime/routes/slack-channel-routes.ts +2 -4
  280. package/src/runtime/routes/workspace-commit-routes.ts +49 -2
  281. package/src/runtime/routes/workspace-routes.ts +11 -12
  282. package/src/runtime/routes/workspace-utils.ts +49 -2
  283. package/src/runtime/services/conversation-serializer.ts +2 -4
  284. package/src/subagent/__tests__/consult-context-gating.test.ts +98 -0
  285. package/src/subagent/__tests__/consult-context-skills.test.ts +82 -0
  286. package/src/subagent/__tests__/consult-context.test.ts +84 -0
  287. package/src/subagent/__tests__/consult-prompt.test.ts +36 -0
  288. package/src/subagent/consult-context.ts +410 -0
  289. package/src/subagent/consult-prompt.ts +38 -5
  290. package/src/subagent/manager.ts +9 -4
  291. package/src/subagent/types.ts +8 -0
  292. package/src/telemetry/telemetry-event-sources.ts +7 -0
  293. package/src/telemetry/telemetry-wire-source.json +1 -1
  294. package/src/telemetry/telemetry-wire.generated.ts +1 -0
  295. package/src/telemetry/turn-events-store.ts +30 -1
  296. package/src/telemetry/types.ts +24 -0
  297. package/src/telemetry/usage-telemetry-reporter.test.ts +48 -0
  298. package/src/tools/browser/pinned-tabs.ts +3 -1
  299. package/src/tools/subagent/spawn.ts +23 -1
  300. package/src/tools/terminal/safe-env.ts +1 -0
  301. package/src/tools/ui-surface/definitions.ts +39 -1
  302. package/src/tools/ui-surface/surface-shape-docs.ts +38 -4
  303. package/src/tools/ui-surface/visual-validation.ts +787 -0
  304. package/src/tools/workflows/run-workflow.test.ts +1 -0
  305. package/src/util/__tests__/short-title.test.ts +229 -0
  306. package/src/util/__tests__/worker-compute.test.ts +65 -0
  307. package/src/util/cgroup-cpu.ts +93 -0
  308. package/src/util/errors.ts +27 -0
  309. package/src/util/process-tree.ts +19 -0
  310. package/src/util/short-title.ts +189 -0
  311. package/src/util/worker-compute.ts +83 -0
  312. package/src/workspace/byok-default-profile-ensure.ts +25 -15
  313. package/src/workspace/git-service.ts +79 -30
  314. package/src/workspace/migrations/137-repair-retired-fireworks-minimax-model-id.ts +131 -0
  315. package/src/workspace/migrations/138-backfill-home-feed-titles.ts +179 -0
  316. package/src/workspace/migrations/139-clear-renamed-cost-profile-label.ts +92 -0
  317. package/src/workspace/migrations/registry.ts +6 -0
  318. package/src/plugins/defaults/memory/substrate/constants.ts +0 -8
@@ -8,7 +8,6 @@ import {
8
8
  import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
9
9
  import {
10
10
  isIncompleteControlMarkerTail,
11
- MINIMIZE_ROOM_MARKER,
12
11
  stripInternalSpeechMarkers,
13
12
  } from "../calls/voice-control-protocol.js";
14
13
  import type {
@@ -59,6 +58,12 @@ import { getToolOwner } from "../tools/registry.js";
59
58
  import { extractSpeakableSegments } from "../tts/speakable-segments.js";
60
59
  import { createAbortReason } from "../util/abort-reasons.js";
61
60
  import { getLogger } from "../util/logger.js";
61
+ import {
62
+ activityLabelForTool,
63
+ approvalActivityLabel,
64
+ dismissesUiSurface,
65
+ revealsUiSurface,
66
+ } from "./activity-label.js";
62
67
  import {
63
68
  createVoiceFrontDecider,
64
69
  type VoiceFrontDecider,
@@ -478,9 +483,38 @@ interface ActiveAssistantTurn {
478
483
  progress: TurnProgressState;
479
484
  assistantCompleted: boolean;
480
485
  ttsDone: boolean;
481
- // Latched when the leg's stream contains MINIMIZE_ROOM_MARKER; consumed
486
+ // Latched when the turn puts something on screen (a ui tool ran); consumed
482
487
  // once at TTS drain, where the minimize_room frame goes out after tts_done.
488
+ // Never set from anything the model says: the reveal is a consequence of
489
+ // showing a surface, not a token the model has to remember.
483
490
  minimizeRequested: boolean;
491
+ // The activity label the client was last told about, so a run of tools that
492
+ // map to the same line sends one frame rather than one per call. Empty means
493
+ // the client believes nothing is running, which is also where a turn ends.
494
+ activityLabel: string;
495
+ // The approval id that went out with it, so the de-duplication covers the
496
+ // whole frame rather than its wording. What the CLIENT believes, as against
497
+ // `pendingApproval`, which is what is true.
498
+ publishedApprovalRequestId: string | null;
499
+ // Set while the turn is blocked on a decision the user has to make, and null
500
+ // when it is not. Suppresses progress narration, whose entire vocabulary
501
+ // ("still on it", "almost there") describes work in flight and would be
502
+ // false here.
503
+ //
504
+ // Carries the request id so a surface that is not the app — the Live
505
+ // Activity's buttons — can answer *that* request rather than whatever is
506
+ // pending by the time the tap arrives, and the wording that named it, which
507
+ // is captured once at the reveal rather than recomputed: a parallel op
508
+ // starting mid-wait moves `currentActivityLabel` on, and the line beside an
509
+ // Approve button must keep naming the thing being approved.
510
+ //
511
+ // The FIRST one, on a turn that leaves two decisions pending at once: the
512
+ // wait is announced once (see `revealRoomForPendingApproval`) and the pair
513
+ // resolves as one. A tap answering the first after it has already been
514
+ // decided is dropped client-side by the id check, which is the safe end of
515
+ // a rare case — the island never silently answers a request other than the
516
+ // one it named.
517
+ pendingApproval: { requestId: string; label: string } | null;
484
518
  // A tts_audio frame actually went out to the client — latches on the first
485
519
  // forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
486
520
  ttsAudioStarted: boolean;
@@ -599,13 +633,16 @@ interface ActiveAssistantTurn {
599
633
  * runs forward from the emitted boundary — not from the last "[" — so
600
634
  * brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
601
635
  * in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
602
- * its terminator. As a side effect the force flush latches the turn's
603
- * `minimizeRequested` when the completed stream ENDS with
604
- * MINIMIZE_ROOM_MARKER terminal position only, matching the prompt's
605
- * "end the reply with it" contract, so a reply whose CONTENT happens to
606
- * contain "[-1]" (an array literal, a temperature) never minimizes the
607
- * room. The marker itself is stripped wherever it appears (the shared
608
- * marker-strip convention), so the latch is the only observable.
636
+ * its terminator.
637
+ *
638
+ * **Markers are stripped, never acted on.** Nothing the model can say
639
+ * minimizes the room any more: that is decided by whether a ui tool ran (see
640
+ * the `tool_use_start` handler), so the reveal cannot depend on the model
641
+ * remembering a token, and a reply whose content happens to contain "[-1]" (an
642
+ * array literal, a temperature) cannot move the room either. No prompt teaches
643
+ * a marker, so this stripping is defense against a model that emits one
644
+ * regardless: an unspoken, unpersisted "[-1]" is the correct handling of a
645
+ * token that now means nothing.
609
646
  */
610
647
  function createControlMarkerHoldback(
611
648
  turn: ActiveAssistantTurn,
@@ -613,13 +650,6 @@ function createControlMarkerHoldback(
613
650
  ): (raw: string, opts?: { force?: boolean }) => void {
614
651
  let emitted = 0;
615
652
  return (raw, opts) => {
616
- if (
617
- opts?.force === true &&
618
- !turn.minimizeRequested &&
619
- raw.trimEnd().endsWith(MINIMIZE_ROOM_MARKER)
620
- ) {
621
- turn.minimizeRequested = true;
622
- }
623
653
  let safeEnd = raw.length;
624
654
  if (opts?.force !== true) {
625
655
  for (
@@ -644,30 +674,30 @@ function createControlMarkerHoldback(
644
674
  // barge-in, the interruption merge note is appended to it (see
645
675
  // buildInterruptionMergeNote) so the model reconciles the interrupted request
646
676
  // with the new utterance.
677
+ // Spoken once when a turn starts waiting on the user's decision. Fixed rather
678
+ // than generated: this is a statement about the system's state, not about the
679
+ // work, and it has to be true every time. Kept in the shape of the progress
680
+ // phrases it displaces (short, neutral, no claim about tools).
681
+ const APPROVAL_PENDING_PHRASE = "I need your okay for that one. Take a look.";
682
+
647
683
  const LIVE_VOICE_CONTROL_PROMPT_BASE =
648
- "You are speaking in a local live voice session. Keep replies brief and conversational. You cannot display cards, forms, or any on-screen UI during the call convey everything in speech. ";
649
-
650
- // MINIMIZE_ROOM_MARKER teaching, appended for the legs that can actually put
651
- // something on screen — the main leg and the escalated leg. The front-door
652
- // (fast) leg never receives it: that leg is toolless, so it has nothing to
653
- // show, and its decision rule promises that apart from a leading verdict
654
- // token every character is spoken verbatim — teaching it the marker would
655
- // contradict that rule and could only produce spurious minimizes.
656
- //
657
- // The teaching deliberately keeps the no-interactive-UI rule intact
658
- // (live-voice turns are non-interactive, JARVIS-1291): the marker does not
659
- // render UI — it asks the client to reveal the screen the call overlay
660
- // covers. The gate in createControlMarkerHoldback strips it from speech, and
661
- // completeTtsForTurn sends the minimize_room frame only after the reply's
662
- // TTS drains, so "end your reply with it" is the whole contract the model
663
- // needs.
684
+ "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. ";
685
+
686
+ // Appended for the legs that can actually put something on screen: the main
687
+ // leg and the escalated leg. The front-door (fast) leg never receives it, for
688
+ // the same reason it never received the marker this replaces: that leg is
689
+ // toolless, so it has nothing to show.
664
690
  //
665
- // The narration guidance is load-bearing: without it, the base prompt's
666
- // convey-everything-in-speech rule reads to the model as "the user can never
667
- // see the screen", and it minimizes the room while telling the user it has
668
- // no way to show them anything ("check it from the app later").
669
- const LIVE_VOICE_MINIMIZE_MARKER_TEACHING =
670
- "The call renders as a full-screen overlay covering the app. If this reply created or changed something on screen worth looking at (an app, a page, a document), you may end the reply with the marker [-1]: after you finish speaking, the overlay minimizes so the user can see the screen while the call continues. The user is in the app and sees what you made the moment the overlay minimizes, so when you end with the marker, speak as if you are showing it to them right now (for example, close with something like: take a look) — never say you cannot show it, that you cannot display it because this is a voice call, or that they should check it later. Use it at most once per reply, only when there is genuinely something new to show, never speak the marker aloud or mention it, and never emit any other bracketed marker. ";
691
+ // **The model no longer asks for the minimize; it is told one is coming.**
692
+ // Revealing the screen used to be a marker the model emitted at the end of a
693
+ // reply, which made "did the user see it" depend on the model remembering a
694
+ // token. It is now a consequence of showing a surface at all: the session
695
+ // latches the minimize when a ui tool runs, and the room opens after the
696
+ // reply's speech drains. What is left for the prompt is the part only the
697
+ // model can get right, which is speaking as though the thing is already in
698
+ // front of the user, because by the time it stops talking it is.
699
+ const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
700
+ "The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. Never emit bracketed markers of any kind. ";
671
701
 
672
702
  // System-level guidance appended to a barge-in turn's control prompt so the
673
703
  // model treats the new utterance as a continuation of the request it was cut
@@ -726,8 +756,8 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
726
756
  }
727
757
 
728
758
  // Assemble a leg's model-facing control prompt: the base live-voice rules,
729
- // the [-1] minimize teaching (withheld from the front-door leg see
730
- // LIVE_VOICE_MINIMIZE_MARKER_TEACHING), the shared no-setup-flows rule, plus
759
+ // the screen-reveal teaching (withheld from the front-door leg, see
760
+ // LIVE_VOICE_SCREEN_REVEAL_TEACHING), the shared no-setup-flows rule, plus
731
761
  // any pending barge-in merge context, completed-continuation context, and/or
732
762
  // the announcement instruction. A turn can carry several (a barge-in follow-up
733
763
  // that also has a continuation result waiting); the notes are model-only and
@@ -738,7 +768,7 @@ function buildVoiceControlPrompt(
738
768
  ): string {
739
769
  let prompt =
740
770
  LIVE_VOICE_CONTROL_PROMPT_BASE +
741
- (leg.frontDoor === true ? "" : LIVE_VOICE_MINIMIZE_MARKER_TEACHING) +
771
+ (leg.frontDoor === true ? "" : LIVE_VOICE_SCREEN_REVEAL_TEACHING) +
742
772
  VOICE_NO_SETUP_FLOWS_RULE;
743
773
  if (turn.interruptedRequest) {
744
774
  prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
@@ -1920,6 +1950,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1920
1950
  "barge_in",
1921
1951
  turn.turnId,
1922
1952
  );
1953
+ // A cancelled turn's tools are abandoned, not finished, so nothing will
1954
+ // arrive to clear the line it left behind.
1955
+ this.publishActivity(turn, "");
1923
1956
  await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
1924
1957
  await this.cancelAssistantTurn("barge_in");
1925
1958
  // Keep the interrupted turn's work alive on a background subagent; the
@@ -2317,6 +2350,172 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2317
2350
  }, drainMs + this.continuationAnnounceSilenceMs);
2318
2351
  }
2319
2352
 
2353
+ /**
2354
+ * The line describing what this turn is doing right now: the newest tool
2355
+ * still running, or nothing when none is.
2356
+ *
2357
+ * Newest-first because parallel ops are the interesting case. A turn that
2358
+ * starts a search and a file read, then finishes the search, is still
2359
+ * reading a file, and a surface that went blank there would claim the turn
2360
+ * had gone idle while it plainly has not.
2361
+ */
2362
+ private currentActivityLabel(turn: ActiveAssistantTurn): string {
2363
+ for (let i = turn.progress.ops.length - 1; i >= 0; i -= 1) {
2364
+ const op = turn.progress.ops[i];
2365
+ if (op !== undefined && op.completedAtMs === undefined) {
2366
+ return activityLabelForTool(op.toolName);
2367
+ }
2368
+ }
2369
+ return "";
2370
+ }
2371
+
2372
+ /**
2373
+ * The line for a turn that is waiting on a decision about its newest running
2374
+ * op.
2375
+ *
2376
+ * The approval gate sits behind `tool_use_start`, so the tool being waited on
2377
+ * is the one the turn last said it was running — which is precisely why the
2378
+ * wait has to be published at all: without it the surfaces keep showing that
2379
+ * tool as *running* for the whole time it is doing nothing of the kind.
2380
+ * Falls back to the tool-less phrase when no op is open, which is the case
2381
+ * for a confirmation raised by a prompter outside the tool pipeline.
2382
+ *
2383
+ * Read once, at the reveal — see `pendingApproval` for why it is then held
2384
+ * rather than recomputed.
2385
+ */
2386
+ private pendingApprovalLabel(turn: ActiveAssistantTurn): string {
2387
+ for (let i = turn.progress.ops.length - 1; i >= 0; i -= 1) {
2388
+ const op = turn.progress.ops[i];
2389
+ if (op !== undefined && op.completedAtMs === undefined) {
2390
+ return approvalActivityLabel(op.toolName);
2391
+ }
2392
+ }
2393
+ return approvalActivityLabel("");
2394
+ }
2395
+
2396
+ /**
2397
+ * Publish whatever the turn's activity line should be right now: the
2398
+ * decision it is waiting on if it is waiting, and its newest running tool if
2399
+ * it is not.
2400
+ *
2401
+ * The single entry point for every caller that would otherwise reach for
2402
+ * `currentActivityLabel` directly. A turn can start and finish other ops
2403
+ * while it is blocked on an approval — a parallel `tool_use_start` or
2404
+ * `tool_result` lands mid-wait — and each of those would otherwise publish a
2405
+ * line composed as if nothing were pending, taking the request id down with
2406
+ * it and retiring the island's buttons while the turn was still waiting.
2407
+ */
2408
+ private refreshActivity(turn: ActiveAssistantTurn): void {
2409
+ if (turn.pendingApproval !== null) {
2410
+ this.publishActivity(
2411
+ turn,
2412
+ turn.pendingApproval.label,
2413
+ turn.pendingApproval.requestId,
2414
+ );
2415
+ return;
2416
+ }
2417
+ this.publishActivity(turn, this.currentActivityLabel(turn));
2418
+ }
2419
+
2420
+ /**
2421
+ * Open the room because a decision is waiting behind it.
2422
+ *
2423
+ * The approval card renders in the app, and the room covers the app, so
2424
+ * without this the turn simply goes quiet: nothing is spoken, nothing is
2425
+ * visible, and the only cue is a call that stopped talking. Sent
2426
+ * immediately rather than latched for the drain the way a shown surface is,
2427
+ * because there is no drain coming. The turn is blocked on this decision,
2428
+ * and the whole point is that the user can make it now.
2429
+ *
2430
+ * The latch is cleared as well, so a turn that also showed a surface does
2431
+ * not send a second minimize once its speech ends; the room is already open.
2432
+ */
2433
+ private revealRoomForPendingApproval(
2434
+ turn: ActiveAssistantTurn,
2435
+ requestId: string,
2436
+ ): void {
2437
+ if (turn.pendingApproval !== null) {
2438
+ return;
2439
+ }
2440
+ turn.pendingApproval = {
2441
+ requestId,
2442
+ label: this.pendingApprovalLabel(turn),
2443
+ };
2444
+ turn.minimizeRequested = false;
2445
+ // Say what the turn is actually doing on the surfaces that are not the
2446
+ // app. Without this the island keeps showing the tool as running for the
2447
+ // whole wait — and it is the surface most likely to be the only one the
2448
+ // user can see, since the case this exists for is a phone put down.
2449
+ // Carrying the request id is what makes the line answerable there rather
2450
+ // than merely accurate.
2451
+ this.refreshActivity(turn);
2452
+ void this.sendFrame(
2453
+ { type: "minimize_room", turnId: turn.turnId },
2454
+ () => !this.isClosed,
2455
+ );
2456
+ // Spoken, because opening the room is only a cue for someone looking at
2457
+ // the screen, and the case this exists for is a phone the user has put
2458
+ // down. One line, not narration: the turn is not working, it is waiting,
2459
+ // and it says which.
2460
+ this.enqueueFillerPhrase(turn, APPROVAL_PENDING_PHRASE);
2461
+ }
2462
+
2463
+ /** Clear the wait once a decision lands, so the turn narrates normally again. */
2464
+ private clearAwaitingApproval(turn: ActiveAssistantTurn): void {
2465
+ turn.pendingApproval = null;
2466
+ // Put the activity line back to whatever the turn resumed doing, and
2467
+ // retire the request id with it, so the island's Approve/Deny buttons go
2468
+ // away the moment the decision is no longer the user's to make — including
2469
+ // when it was made somewhere else entirely (the card in the app, the
2470
+ // 45-second fallback, a superseding message).
2471
+ this.refreshActivity(turn);
2472
+ }
2473
+
2474
+ /**
2475
+ * Tell the client what the turn is doing, if it changed.
2476
+ *
2477
+ * De-duplicated for the same reason the Live Activity reporter de-duplicates
2478
+ * phases: this frame reaches ActivityKit, whose update budget is finite and
2479
+ * whose overflow is dropped silently, and a turn that runs four file reads in
2480
+ * a row would otherwise spend that budget restating one line.
2481
+ *
2482
+ * Fire-and-forget. An activity label is a flourish, and nothing about the
2483
+ * conversation may wait on one.
2484
+ *
2485
+ * Callers whose line depends on turn state go through
2486
+ * {@link refreshActivity}; this is called directly only to clear the line
2487
+ * outright, which a cancelled or finished turn does regardless of what it
2488
+ * was waiting on.
2489
+ */
2490
+ private publishActivity(
2491
+ turn: ActiveAssistantTurn,
2492
+ label: string,
2493
+ approvalRequestId?: string,
2494
+ ): void {
2495
+ // De-duplicated on the request id as well as the wording. The two move
2496
+ // independently: a wait can be entered and left without the tool line
2497
+ // changing at all, and a label-only check would swallow the frame that
2498
+ // retires the approval — leaving the island's buttons up with nothing
2499
+ // behind them.
2500
+ if (
2501
+ turn.activityLabel === label &&
2502
+ turn.publishedApprovalRequestId === (approvalRequestId ?? null)
2503
+ ) {
2504
+ return;
2505
+ }
2506
+ turn.activityLabel = label;
2507
+ turn.publishedApprovalRequestId = approvalRequestId ?? null;
2508
+ void this.sendFrame(
2509
+ {
2510
+ type: "activity",
2511
+ turnId: turn.turnId,
2512
+ label,
2513
+ ...(approvalRequestId !== undefined ? { approvalRequestId } : {}),
2514
+ },
2515
+ () => !this.isClosed,
2516
+ );
2517
+ }
2518
+
2320
2519
  private clearActiveAssistantTurn(token: symbol): void {
2321
2520
  if (this.activeAssistantTurn?.token !== token) {
2322
2521
  return;
@@ -3409,6 +3608,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3409
3608
  assistantCompleted: false,
3410
3609
  ttsDone: false,
3411
3610
  minimizeRequested: false,
3611
+ activityLabel: "",
3612
+ publishedApprovalRequestId: null,
3613
+ pendingApproval: null,
3412
3614
  ttsAudioStarted: false,
3413
3615
  finalized: false,
3414
3616
  speculativePending: opts?.speculative === true,
@@ -3610,6 +3812,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3610
3812
  voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
3611
3813
  ...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
3612
3814
  }),
3815
+ onApprovalPending: (requestId) => {
3816
+ this.revealRoomForPendingApproval(activeTurn, requestId);
3817
+ },
3818
+ onApprovalsResolved: () => {
3819
+ this.clearAwaitingApproval(activeTurn);
3820
+ },
3613
3821
  content: leg.content,
3614
3822
  isInbound: true,
3615
3823
  launchedAtMs: activeTurn.launchedAtMs,
@@ -3835,6 +4043,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3835
4043
  });
3836
4044
  current.progress.opsSinceNarration += 1;
3837
4045
  current.progress.stateEpoch += 1;
4046
+ this.refreshActivity(current);
3838
4047
  log.debug({ turnId, toolName }, "Live voice turn started tool use");
3839
4048
  // Definitive tool use means the turn is guaranteed slow: speak
3840
4049
  // the floor-holding ack now instead of waiting out the
@@ -3888,6 +4097,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3888
4097
  }
3889
4098
  }
3890
4099
  current.progress.stateEpoch += 1;
4100
+ // Showing a surface implies revealing it: the room is a
4101
+ // full-screen overlay, so a surface rendered behind it is a
4102
+ // surface nobody sees.
4103
+ //
4104
+ // **Latched on the result, not the tool start.** A ui call can be
4105
+ // rejected (no `surface_type`, an empty card, a client that never
4106
+ // acks), and a reveal driven by the attempt would minimize the room
4107
+ // to show nothing at all. Only a call that came back without an
4108
+ // error actually put something on screen.
4109
+ //
4110
+ // A dismissal clears it again, so a turn that shows a surface and
4111
+ // then takes it away does not reveal an empty screen. Last write
4112
+ // wins, which is the right reading of "what did this turn leave up"
4113
+ // without tracking surfaces individually.
4114
+ if (event.isError !== true) {
4115
+ if (revealsUiSurface(event.toolName)) {
4116
+ current.minimizeRequested = true;
4117
+ } else if (dismissesUiSurface(event.toolName)) {
4118
+ current.minimizeRequested = false;
4119
+ }
4120
+ }
4121
+ this.refreshActivity(current);
3891
4122
  this.maybeNarrateProgress(current, trigger);
3892
4123
  },
3893
4124
  },
@@ -4259,6 +4490,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4259
4490
  return (
4260
4491
  this.isActiveAssistantTurn(turn.token) &&
4261
4492
  !turn.assistantCompleted &&
4493
+ // Nothing is in flight while a decision is pending, so every phrase
4494
+ // narration has would be a lie about who the call is waiting on. The
4495
+ // turn says so once, when it starts waiting, and is quiet after that.
4496
+ turn.pendingApproval === null &&
4262
4497
  this.turnAudioIdle(turn)
4263
4498
  );
4264
4499
  }
@@ -4569,6 +4804,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4569
4804
  !this.isClosed,
4570
4805
  );
4571
4806
 
4807
+ // The turn is over, so nothing is running. Ordinarily the line is
4808
+ // already clear (the last tool_result cleared it); this is the net
4809
+ // for a turn that ends with an op still open, which would otherwise
4810
+ // leave the last tool it touched on screen through the next silence.
4811
+ this.publishActivity(currentTurn, "");
4812
+
4572
4813
  // Drain-scoped minimize: the latched marker is consumed here, after
4573
4814
  // the turn's speech has fully drained — never mid-speech, never for
4574
4815
  // a barged-in turn, at most once per turn.
@@ -18,6 +18,7 @@ const _LIVE_VOICE_SERVER_FRAME_TYPES = [
18
18
  "stt_partial",
19
19
  "stt_final",
20
20
  "thinking",
21
+ "activity",
21
22
  "assistant_text_delta",
22
23
  "tts_audio",
23
24
  "tts_done",
@@ -217,6 +218,38 @@ export interface LiveVoiceThinkingServerFrame extends LiveVoiceServerFrameBase {
217
218
  readonly turnId: string;
218
219
  }
219
220
 
221
+ /**
222
+ * What the assistant is doing inside a turn, as one short user-facing line.
223
+ *
224
+ * Exists because a live-voice turn can go silent for a long time while it
225
+ * works, and the surfaces that show the session (the iOS Live Activity, and
226
+ * the room) can otherwise only say "Thinking...". The label is composed here
227
+ * rather than by each surface for the same reason phase wording is composed
228
+ * once: the Live Activity has two independent drivers (this socket and an APNs
229
+ * push the daemon dispatches), they must carry identical content, and the only
230
+ * way to guarantee that is for both to be handed the same string.
231
+ *
232
+ * An empty `label` means "no current activity", which is what a turn's end
233
+ * sends. Emitted only on change, never per tool result.
234
+ */
235
+ export interface LiveVoiceActivityServerFrame extends LiveVoiceServerFrameBase {
236
+ readonly type: "activity";
237
+ readonly turnId: string;
238
+ readonly label: string;
239
+ /**
240
+ * The confirmation this turn is blocked on, when the label describes a wait
241
+ * rather than work in flight. Absent otherwise.
242
+ *
243
+ * It travels so that a surface outside the app — the Live Activity's
244
+ * Approve/Deny buttons — can answer the request it was drawn against rather
245
+ * than whatever is pending by the time the press lands. Content on a Lock
246
+ * Screen can be seconds old, and a decision is the one thing that must not
247
+ * be re-pointed when it is: the id lets a client drop a press aimed at a
248
+ * request already answered, timed out, or superseded.
249
+ */
250
+ readonly approvalRequestId?: string;
251
+ }
252
+
220
253
  export interface LiveVoiceAssistantTextDeltaServerFrame extends LiveVoiceServerFrameBase {
221
254
  readonly type: "assistant_text_delta";
222
255
  readonly text: string;
@@ -321,6 +354,7 @@ export type LiveVoiceServerFrame =
321
354
  | LiveVoiceSttPartialServerFrame
322
355
  | LiveVoiceSttFinalServerFrame
323
356
  | LiveVoiceThinkingServerFrame
357
+ | LiveVoiceActivityServerFrame
324
358
  | LiveVoiceAssistantTextDeltaServerFrame
325
359
  | LiveVoiceTtsAudioServerFrame
326
360
  | LiveVoiceTtsDoneServerFrame
@@ -341,6 +375,7 @@ export type LiveVoiceServerFramePayload =
341
375
  | WithoutSeq<LiveVoiceSttPartialServerFrame>
342
376
  | WithoutSeq<LiveVoiceSttFinalServerFrame>
343
377
  | WithoutSeq<LiveVoiceThinkingServerFrame>
378
+ | WithoutSeq<LiveVoiceActivityServerFrame>
344
379
  | WithoutSeq<LiveVoiceAssistantTextDeltaServerFrame>
345
380
  | WithoutSeq<LiveVoiceTtsAudioServerFrame>
346
381
  | WithoutSeq<LiveVoiceTtsDoneServerFrame>
@@ -31,12 +31,13 @@ mock.module("../../../../contacts/contacts-write.js", () => ({
31
31
  }));
32
32
 
33
33
  import {
34
- __resetSlackUserInfoCacheForTests,
34
+ __resetSlackMentionCachesForTests,
35
35
  slackProvider,
36
36
  } from "../adapter.js";
37
37
 
38
38
  const originalFetch = globalThis.fetch;
39
39
  let userInfoCalls: string[] = [];
40
+ let channelInfoCalls: string[] = [];
40
41
 
41
42
  function installFetchStub() {
42
43
  globalThis.fetch = (async (
@@ -63,7 +64,49 @@ function fakeSlackResponse(url: string): Record<string, unknown> {
63
64
  const parsed = new URL(url);
64
65
  const method = parsed.pathname.split("/").at(-1);
65
66
 
67
+ if (method === "conversations.info") {
68
+ const channelId = parsed.searchParams.get("channel") ?? "";
69
+ channelInfoCalls.push(channelId);
70
+ if (channelId === "CPRODMODELS1") {
71
+ return {
72
+ ok: true,
73
+ channel: { id: "CPRODMODELS1", name: "prod-models" },
74
+ };
75
+ }
76
+ return { ok: false, error: "channel_not_found" };
77
+ }
78
+
66
79
  if (method === "conversations.history") {
80
+ if (parsed.searchParams.get("channel") === "C_CHANNEL_MENTIONS") {
81
+ return {
82
+ ok: true,
83
+ has_more: false,
84
+ messages: [
85
+ {
86
+ type: "message",
87
+ ts: "1700000009.001000",
88
+ user: "USENDER",
89
+ text: "post in <#CPRODMODELS1> and <#COTHER1|other-name> please",
90
+ },
91
+ ],
92
+ };
93
+ }
94
+
95
+ if (parsed.searchParams.get("channel") === "C_PRIVATE_MENTION") {
96
+ return {
97
+ ok: true,
98
+ has_more: false,
99
+ messages: [
100
+ {
101
+ type: "message",
102
+ ts: "1700000010.001100",
103
+ user: "USENDER",
104
+ text: "see <#CINVISIBLE1> for details",
105
+ },
106
+ ],
107
+ };
108
+ }
109
+
67
110
  if (parsed.searchParams.get("channel") === "C_USERINFO_FAIL") {
68
111
  return {
69
112
  ok: true,
@@ -318,8 +361,9 @@ function makeOAuthConnection(
318
361
 
319
362
  describe("Slack adapter mention rendering", () => {
320
363
  beforeEach(async () => {
321
- __resetSlackUserInfoCacheForTests();
364
+ __resetSlackMentionCachesForTests();
322
365
  userInfoCalls = [];
366
+ channelInfoCalls = [];
323
367
  getSecureKeyAsyncMock.mockReset();
324
368
  getSecureKeyAsyncMock.mockImplementation(async (key: string) => {
325
369
  if (key === credentialKey("slack_channel", "bot_token")) {
@@ -516,6 +560,44 @@ describe("Slack adapter mention rendering", () => {
516
560
  });
517
561
  });
518
562
 
563
+ test("getHistory renders inline channel mentions without rebinding the message's own channel", async () => {
564
+ // A message read from channel A that mentions channels B and C: the
565
+ // message stays attributed to A while the inline mentions render B/C.
566
+ const messages = await slackProvider.getHistory(
567
+ undefined,
568
+ "C_CHANNEL_MENTIONS",
569
+ );
570
+
571
+ expect(messages).toHaveLength(1);
572
+ // Bare token resolves via conversations.info; pipe token uses its own
573
+ // embedded label with no lookup.
574
+ expect(messages[0].text).toBe(
575
+ "post in #prod-models and #other-name please",
576
+ );
577
+ expect(messages[0].conversationId).toBe("C_CHANNEL_MENTIONS");
578
+ expect(channelInfoCalls).toEqual(["CPRODMODELS1"]);
579
+ });
580
+
581
+ test("getHistory falls back to #unknown-channel for channels this auth cannot see, and caches the failure", async () => {
582
+ const first = await slackProvider.getHistory(
583
+ undefined,
584
+ "C_PRIVATE_MENTION",
585
+ );
586
+ const second = await slackProvider.getHistory(
587
+ undefined,
588
+ "C_PRIVATE_MENTION",
589
+ );
590
+
591
+ expect(first[0].text).toBe("see #unknown-channel for details");
592
+ expect(second[0].text).toBe("see #unknown-channel for details");
593
+ expect(first[0].text).not.toContain("CINVISIBLE1");
594
+ // channel_not_found is a definitive answer for this auth: one lookup,
595
+ // not one per history read.
596
+ expect(channelInfoCalls.filter((id) => id === "CINVISIBLE1")).toHaveLength(
597
+ 1,
598
+ );
599
+ });
600
+
519
601
  test("getThreadReplies renders Slack user mentions for model-facing text without changing sender identity", async () => {
520
602
  const messages = await slackProvider.getThreadReplies!(
521
603
  undefined,