@vellumai/assistant 0.11.7 → 0.11.8-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/ARCHITECTURE.md +10 -10
  2. package/Dockerfile +1 -0
  3. package/docs/architecture/security.md +1 -1
  4. package/docs/error-handling.md +1 -2
  5. package/docs/guardian-request-flow.md +51 -5
  6. package/docs/trusted-contact-access.md +13 -12
  7. package/knip.json +1 -0
  8. package/node_modules/@vellumai/avatar-catalog/package.json +19 -0
  9. package/{src/avatar/character-components.ts → node_modules/@vellumai/avatar-catalog/src/catalog.ts} +12 -44
  10. package/node_modules/@vellumai/avatar-catalog/src/colors.ts +19 -0
  11. package/node_modules/@vellumai/avatar-catalog/src/index.ts +25 -0
  12. package/node_modules/@vellumai/avatar-catalog/src/types.ts +47 -0
  13. package/node_modules/@vellumai/avatar-catalog/tsconfig.json +20 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
  18. package/node_modules/@vellumai/credential-storage/src/static-credentials.ts +43 -12
  19. package/node_modules/@vellumai/environments/package.json +2 -1
  20. package/node_modules/@vellumai/environments/src/__tests__/test-path-guard.test.ts +89 -0
  21. package/node_modules/@vellumai/environments/src/test-path-guard.ts +113 -0
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
  26. package/node_modules/@vellumai/gateway-client/src/__tests__/guardian-request-contract.test.ts +8 -18
  27. package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +46 -0
  28. package/node_modules/@vellumai/gateway-client/src/__tests__/invite-contract.test.ts +1 -0
  29. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +46 -0
  30. package/node_modules/@vellumai/gateway-client/src/guardian-request-contract.ts +62 -21
  31. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +14 -0
  32. package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +53 -0
  33. package/node_modules/@vellumai/gateway-client/src/index.ts +11 -5
  34. package/node_modules/@vellumai/gateway-client/src/invite-contract.ts +1 -0
  35. package/node_modules/@vellumai/service-contracts/package.json +1 -0
  36. package/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
  37. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  38. package/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
  39. package/openapi.yaml +481 -9
  40. package/package.json +3 -1
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/src/__tests__/agent-loop-callsite-precedence.test.ts +16 -0
  43. package/src/__tests__/agent-loop-compacted-history-transform.test.ts +304 -0
  44. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +3 -0
  45. package/src/__tests__/agent-wake-override-profile.test.ts +3 -0
  46. package/src/__tests__/anthropic-provider.test.ts +56 -0
  47. package/src/__tests__/approval-cascade.test.ts +0 -13
  48. package/src/__tests__/attachments-store.test.ts +117 -6
  49. package/src/__tests__/attachments.test.ts +122 -0
  50. package/src/__tests__/bundled-asset.test.ts +22 -1
  51. package/src/__tests__/channel-approval.test.ts +14 -7
  52. package/src/__tests__/channel-approvals.test.ts +3 -3
  53. package/src/__tests__/channel-delivery-store.test.ts +58 -0
  54. package/src/__tests__/channel-inbound-disk-pressure.test.ts +1 -0
  55. package/src/__tests__/channel-invite-transport.test.ts +1 -0
  56. package/src/__tests__/channel-readiness-slack-remote.test.ts +3 -3
  57. package/src/__tests__/compaction-events.test.ts +0 -12
  58. package/src/__tests__/compactor-call-site-logging.test.ts +7 -1
  59. package/src/__tests__/compactor-retained-image-validation.test.ts +15 -0
  60. package/src/__tests__/config-schema.test.ts +35 -0
  61. package/src/__tests__/contacts-write.test.ts +141 -3
  62. package/src/__tests__/conversation-abort-tool-results.test.ts +0 -13
  63. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +0 -12
  64. package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -13
  65. package/src/__tests__/conversation-agent-loop.test.ts +95 -13
  66. package/src/__tests__/conversation-confirmation-signals.test.ts +0 -13
  67. package/src/__tests__/conversation-load-history-repair.test.ts +418 -0
  68. package/src/__tests__/conversation-media-retry.test.ts +33 -2
  69. package/src/__tests__/conversation-pairing.test.ts +54 -0
  70. package/src/__tests__/conversation-process-callsite.test.ts +0 -12
  71. package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -13
  72. package/src/__tests__/conversation-queue.test.ts +4 -13
  73. package/src/__tests__/conversation-retry-route.test.ts +5 -0
  74. package/src/__tests__/conversation-routes-disk-view.test.ts +1 -1
  75. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +1 -1
  76. package/src/__tests__/conversation-routes-hidden-queue.test.ts +1 -1
  77. package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -1
  78. package/src/__tests__/conversation-sight-frames.test.ts +615 -0
  79. package/src/__tests__/conversation-slash-queue.test.ts +0 -13
  80. package/src/__tests__/conversation-slash-unknown.test.ts +0 -13
  81. package/src/__tests__/conversation-speed-override.test.ts +0 -12
  82. package/src/__tests__/conversation-store-ephemeral.test.ts +0 -12
  83. package/src/__tests__/conversation-summarize-route.test.ts +5 -0
  84. package/src/__tests__/conversation-summarize-up-to.test.ts +102 -0
  85. package/src/__tests__/conversation-wait-for-idle.test.ts +0 -12
  86. package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -5
  87. package/src/__tests__/conversation-workspace-injection.test.ts +0 -17
  88. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -17
  89. package/src/__tests__/credential-health-service.test.ts +45 -0
  90. package/src/__tests__/credential-record-retire.test.ts +256 -0
  91. package/src/__tests__/credential-record-write-through.test.ts +8 -4
  92. package/src/__tests__/db-conversation-fork-lineage-migration.test.ts +0 -4
  93. package/src/__tests__/db-conversation-inference-profile-migration.test.ts +0 -4
  94. package/src/__tests__/db-llm-request-log-provider-migration.test.ts +0 -8
  95. package/src/__tests__/delete-propagation.test.ts +121 -3
  96. package/src/__tests__/evict-conversations-for-reload.test.ts +46 -6
  97. package/src/__tests__/file-ops-service.test.ts +8 -3
  98. package/src/__tests__/gemini-provider.test.ts +64 -0
  99. package/src/__tests__/guardian-action-service-scope.test.ts +118 -0
  100. package/src/__tests__/guardian-card-withdrawal.test.ts +299 -5
  101. package/src/__tests__/guardian-expiry-notifier.test.ts +117 -13
  102. package/src/__tests__/guardian-gateway-sim.test.ts +119 -0
  103. package/src/__tests__/guardian-gateway-sim.ts +56 -20
  104. package/src/__tests__/guardian-reply-router-answer-mode.test.ts +6 -0
  105. package/src/__tests__/guardian-request-status-sync.test.ts +162 -0
  106. package/src/__tests__/guardian-wait-controller.test.ts +6 -0
  107. package/src/__tests__/headless-browser-mode.test.ts +27 -0
  108. package/src/__tests__/helpers/mock-conversation.ts +2 -0
  109. package/src/__tests__/list-messages-system-card.test.ts +70 -0
  110. package/src/__tests__/llm-catalog-parity.test.ts +2 -0
  111. package/src/__tests__/managed-profile-guard.test.ts +2 -2
  112. package/src/__tests__/media-resolve-image-validation.test.ts +106 -0
  113. package/src/__tests__/media-stream-server-integration.test.ts +6 -0
  114. package/src/__tests__/non-member-access-request.test.ts +12 -0
  115. package/src/__tests__/notification-discord-adapter.test.ts +76 -4
  116. package/src/__tests__/oauth-commands-routes.test.ts +21 -0
  117. package/src/__tests__/openai-provider.test.ts +63 -0
  118. package/src/__tests__/outbound-slack-persistence.test.ts +114 -2
  119. package/src/__tests__/persist-unsendable-image-downscale.test.ts +63 -1
  120. package/src/__tests__/persist-unsendable-image.test.ts +19 -0
  121. package/src/__tests__/plugin-app-serve-routes.test.ts +2 -2
  122. package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -0
  123. package/src/__tests__/process-message-background-slack.test.ts +1 -1
  124. package/src/__tests__/process-message-display-content.test.ts +1 -1
  125. package/src/__tests__/processing-flag-persist-failure.test.ts +0 -5
  126. package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +0 -17
  127. package/src/__tests__/qdrant-manager.test.ts +6 -0
  128. package/src/__tests__/reaction-persistence.test.ts +58 -62
  129. package/src/__tests__/retry-thinking-adaptive-only.test.ts +67 -0
  130. package/src/__tests__/server-history-render.test.ts +22 -0
  131. package/src/__tests__/sight-frame-retention.test.ts +362 -0
  132. package/src/__tests__/slack-reaction-approvals.test.ts +44 -24
  133. package/src/__tests__/slack-reaction-guardian-approval.test.ts +29 -1
  134. package/src/__tests__/subagent-fork-notifications.test.ts +2 -0
  135. package/src/__tests__/subagent-manager-notify.test.ts +2 -0
  136. package/src/__tests__/subagent-notify-parent.test.ts +2 -0
  137. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +85 -0
  138. package/src/__tests__/subagent-tool-gate-mode.test.ts +4 -4
  139. package/src/__tests__/terminal-tools.test.ts +9 -9
  140. package/src/__tests__/tool-result-spool.test.ts +5 -8
  141. package/src/__tests__/usage-attribution.test.ts +26 -0
  142. package/src/acp/__tests__/acp-auth-marker-store.test.ts +191 -0
  143. package/src/acp/__tests__/acp-claude-oauth.test.ts +262 -0
  144. package/src/acp/__tests__/acp-connect-card-state.test.ts +17 -0
  145. package/src/acp/__tests__/helpers/acp-history-db.ts +10 -3
  146. package/src/acp/acp-auth-marker-store.ts +196 -0
  147. package/src/acp/acp-claude-oauth.ts +154 -4
  148. package/src/acp/acp-connect-card-state.ts +49 -8
  149. package/src/acp/agent-process.ts +1 -0
  150. package/src/acp/auto-install.ts +1 -0
  151. package/src/acp/client-handler.ts +1 -0
  152. package/src/acp/prepare-agent-env.ts +79 -1
  153. package/src/acp/resolve-agent.ts +12 -0
  154. package/src/acp/session-manager.test.ts +183 -0
  155. package/src/acp/session-manager.ts +110 -6
  156. package/src/acp/types.ts +17 -0
  157. package/src/agent/attachments.ts +7 -2
  158. package/src/agent/loop.ts +59 -1
  159. package/src/api/events/contact-form-closed.ts +32 -0
  160. package/src/api/events/contact-record-request.ts +63 -0
  161. package/src/api/events/contact-request.ts +10 -3
  162. package/src/api/index.ts +20 -0
  163. package/src/api/responses/conversation-message.ts +20 -0
  164. package/src/api/responses/home.ts +101 -0
  165. package/src/approvals/AGENTS.md +1 -1
  166. package/src/approvals/guardian-card-withdrawal.ts +127 -30
  167. package/src/approvals/guardian-decision-primitive.ts +9 -0
  168. package/src/approvals/guardian-expiry-notifier.ts +22 -16
  169. package/src/approvals/guardian-request-status-sync.ts +98 -0
  170. package/src/avatar/svg-compositor.ts +1 -1
  171. package/src/avatar/traits-png-sync.ts +1 -1
  172. package/src/bundler/app-compiler.ts +1 -0
  173. package/src/bundler/compiler-tools.ts +1 -0
  174. package/src/bundler/package-resolver.ts +1 -0
  175. package/src/calls/__tests__/voice-session-bridge.test.ts +183 -3
  176. package/src/calls/guardian-action-sweep.ts +6 -1
  177. package/src/calls/voice-session-bridge.ts +46 -1
  178. package/src/channels/__tests__/gateway-guardian-requests.test.ts +13 -18
  179. package/src/channels/gateway-guardian-requests.ts +37 -18
  180. package/src/cli/__tests__/contacts-record-prompt-cli.test.ts +329 -0
  181. package/src/cli/commands/__tests__/browser.test.ts +65 -1
  182. package/src/cli/commands/browser.ts +30 -10
  183. package/src/cli/commands/contacts.help.ts +173 -11
  184. package/src/cli/commands/contacts.ts +343 -4
  185. package/src/cli/commands/db/refresh.ts +1 -0
  186. package/src/cli/commands/oauth/index.help.ts +6 -3
  187. package/src/cli/commands/oauth/request.test.ts +37 -0
  188. package/src/cli/commands/oauth/request.ts +7 -6
  189. package/src/cli/lib/bundled-marketplace.json +1 -1
  190. package/src/cli/lib/install-from-github.ts +20 -19
  191. package/src/cli/lib/install-plugin-dependencies.ts +3 -5
  192. package/src/cli/lib/merge-plugin-tree.ts +1 -0
  193. package/src/cli/lib/publish-plugin.ts +4 -1
  194. package/src/config/__tests__/balanced-model-experiment.test.ts +63 -31
  195. package/src/config/__tests__/default-profile-catalog.test.ts +2 -2
  196. package/src/config/balanced-model-experiment.ts +15 -9
  197. package/src/config/bundled-skills/contacts/SKILL.md +61 -18
  198. package/src/config/default-profile-catalog.ts +12 -11
  199. package/src/config/feature-flag-registry.json +22 -13
  200. package/src/config/schema.ts +2 -0
  201. package/src/config/schemas/__tests__/live-voice.test.ts +13 -0
  202. package/src/config/schemas/live-voice.ts +7 -0
  203. package/src/config/schemas/sight.ts +45 -0
  204. package/src/config/skills.ts +2 -1
  205. package/src/contacts/__tests__/mirror-reconciler.test.ts +352 -0
  206. package/src/contacts/contact-store.ts +47 -4
  207. package/src/contacts/contacts-write.ts +5 -2
  208. package/src/contacts/mirror-reconciler.ts +210 -0
  209. package/src/contacts/mirror-tombstones.ts +42 -0
  210. package/src/contacts/notify-contacts-changed.ts +11 -2
  211. package/src/context/compactor.ts +47 -19
  212. package/src/context/tool-result-spool.ts +21 -14
  213. package/src/conversations/message-consolidation.ts +2 -2
  214. package/src/credential-health/credential-health-service.ts +37 -6
  215. package/src/daemon/conversation-agent-loop-handlers.ts +19 -2
  216. package/src/daemon/conversation-agent-loop.ts +100 -1
  217. package/src/daemon/conversation-media-retry.ts +11 -6
  218. package/src/daemon/conversation-messaging.ts +293 -30
  219. package/src/daemon/conversation-runtime-assembly.ts +67 -0
  220. package/src/daemon/conversation-sight-frames.ts +187 -0
  221. package/src/daemon/conversation-store.ts +18 -28
  222. package/src/daemon/conversation-surfaces.ts +12 -2
  223. package/src/daemon/conversation-tool-setup.ts +39 -10
  224. package/src/daemon/conversation.ts +159 -16
  225. package/src/daemon/daemon-control.ts +6 -17
  226. package/src/daemon/handlers/config-model.test.ts +1 -0
  227. package/src/daemon/handlers/conversations.ts +10 -11
  228. package/src/daemon/install-assistant-command.ts +1 -0
  229. package/src/daemon/lifecycle.ts +10 -13
  230. package/src/daemon/message-types/sync.ts +4 -0
  231. package/src/daemon/process-message.ts +2 -17
  232. package/src/daemon/reaction-history-render.test.ts +162 -0
  233. package/src/daemon/reaction-history-render.ts +115 -0
  234. package/src/daemon/reaction-record.test.ts +162 -0
  235. package/src/daemon/reaction-record.ts +83 -0
  236. package/src/daemon/shutdown-handlers.ts +0 -2
  237. package/src/daemon/summarize-boundary.ts +12 -0
  238. package/src/daemon/video-thumbnail.ts +1 -1
  239. package/src/home/feed-types.ts +4 -0
  240. package/src/home/feed-writer.ts +25 -0
  241. package/src/live-voice/__tests__/live-voice-attach-frame.test.ts +679 -0
  242. package/src/live-voice/__tests__/live-voice-attach-image.test.ts +8 -5
  243. package/src/live-voice/__tests__/live-voice-photo.test.ts +637 -1
  244. package/src/live-voice/__tests__/live-voice-sight-frame-inline.test.ts +787 -0
  245. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +347 -0
  246. package/src/live-voice/__tests__/live-voice-vad.test.ts +109 -0
  247. package/src/live-voice/live-voice-photo.ts +420 -65
  248. package/src/live-voice/live-voice-session.ts +311 -27
  249. package/src/live-voice/protocol.ts +147 -11
  250. package/src/messaging/provider-message-metadata.ts +7 -0
  251. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +113 -1
  252. package/src/messaging/providers/channel-transport.ts +37 -0
  253. package/src/messaging/providers/discord/api.ts +4 -4
  254. package/src/messaging/providers/discord/send.test.ts +177 -2
  255. package/src/messaging/providers/discord/send.ts +197 -10
  256. package/src/messaging/providers/discord/transport.ts +10 -0
  257. package/src/messaging/providers/index.ts +42 -0
  258. package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +129 -3
  259. package/src/messaging/providers/slack/__tests__/auth.test.ts +35 -0
  260. package/src/messaging/providers/slack/__tests__/message-content.test.ts +152 -0
  261. package/src/messaging/providers/slack/adapter.ts +112 -27
  262. package/src/messaging/providers/slack/api.test.ts +10 -0
  263. package/src/messaging/providers/slack/auth.ts +32 -6
  264. package/src/messaging/providers/slack/client.ts +0 -15
  265. package/src/messaging/providers/slack/message-content.ts +137 -0
  266. package/src/messaging/providers/slack/send.test.ts +43 -2
  267. package/src/messaging/providers/slack/send.ts +18 -5
  268. package/src/messaging/providers/slack/transport.ts +10 -0
  269. package/src/messaging/providers/slack/types.ts +21 -2
  270. package/src/messaging/providers/telegram-bot/adapter.ts +5 -1
  271. package/src/messaging/providers/telegram-bot/send.test.ts +49 -2
  272. package/src/messaging/providers/telegram-bot/send.ts +41 -1
  273. package/src/messaging/providers/telegram-bot/transport.ts +10 -0
  274. package/src/messaging/providers/telegram-bot/withdraw.ts +13 -5
  275. package/src/messaging/reaction-envelopes.ts +79 -0
  276. package/src/monitoring/control.ts +2 -2
  277. package/src/monitoring/page-cache.ts +5 -1
  278. package/src/notifications/AGENTS.md +1 -1
  279. package/src/notifications/__tests__/guardian-feed-projection.test.ts +360 -0
  280. package/src/notifications/__tests__/home-feed-side-effect.test.ts +8 -0
  281. package/src/notifications/adapters/discord.ts +62 -9
  282. package/src/notifications/conversation-pairing.ts +31 -4
  283. package/src/notifications/emit-signal.ts +8 -5
  284. package/src/notifications/guardian-delivery-recorder.ts +47 -6
  285. package/src/notifications/guardian-feed-projection.ts +492 -0
  286. package/src/notifications/home-feed-side-effect.ts +59 -11
  287. package/src/oauth/byo-connection.test.ts +22 -0
  288. package/src/oauth/byo-connection.ts +11 -5
  289. package/src/oauth/connection.ts +3 -2
  290. package/src/oauth/platform-connection.test.ts +35 -0
  291. package/src/oauth/platform-connection.ts +18 -13
  292. package/src/oauth/scope-utils.ts +48 -0
  293. package/src/oauth/seed-providers.ts +12 -0
  294. package/src/outbound-proxy/certs.ts +5 -4
  295. package/src/permissions/question-guardian-request.test.ts +7 -1
  296. package/src/persistence/attachments-store.ts +134 -31
  297. package/src/persistence/conversation-crud.ts +137 -4
  298. package/src/persistence/conversation-types.ts +109 -0
  299. package/src/persistence/db-connection.ts +3 -21
  300. package/src/persistence/delivery-crud.ts +42 -12
  301. package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +15 -2
  302. package/src/persistence/embeddings/embedding-local.ts +7 -112
  303. package/src/persistence/embeddings/qdrant-manager.ts +4 -1
  304. package/src/persistence/llm-request-log-store.ts +1 -1
  305. package/src/persistence/migrations/370-acp-session-history-auth-error-code.ts +36 -0
  306. package/src/persistence/migrations/371-acp-session-history-auth-error-credential.ts +40 -0
  307. package/src/persistence/migrations/372-create-acp-refused-credentials.ts +34 -0
  308. package/src/persistence/migrations/373-acp-auth-marker-index.ts +27 -0
  309. package/src/persistence/migrations/__tests__/370-acp-session-history-auth-error-code.test.ts +65 -0
  310. package/src/persistence/migrations/__tests__/371-acp-session-history-auth-error-credential.test.ts +68 -0
  311. package/src/persistence/migrations/__tests__/372-create-acp-refused-credentials.test.ts +64 -0
  312. package/src/persistence/migrations/__tests__/373-acp-auth-marker-index.test.ts +82 -0
  313. package/src/persistence/schema/acp.ts +25 -0
  314. package/src/persistence/steps.ts +8 -0
  315. package/src/platform/platform-patch-queue.test.ts +43 -41
  316. package/src/plugin-api/index.ts +12 -0
  317. package/src/plugins/defaults/compaction/window-manager.ts +2 -0
  318. package/src/plugins/defaults/image-recovery/recover.ts +12 -1
  319. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +166 -1
  320. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +3 -0
  321. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +3 -0
  322. package/src/plugins/defaults/memory/jobs-worker.ts +1 -2
  323. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +53 -9
  324. package/src/plugins/defaults/memory/src/__tests__/memory-worker-routes.test.ts +4 -7
  325. package/src/plugins/defaults/memory/src/memory-worker-routes.ts +1 -1
  326. package/src/plugins/defaults/memory/v1/job-handlers/index-maintenance.ts +15 -1
  327. package/src/plugins/defaults/memory/worker-control.ts +1 -2
  328. package/src/providers/content-block-schema.ts +1 -0
  329. package/src/providers/fireworks/client.ts +9 -1
  330. package/src/providers/gemini/client.ts +22 -15
  331. package/src/providers/media-resolve.ts +24 -10
  332. package/src/providers/model-catalog.ts +91 -0
  333. package/src/providers/openai/chat-completions-provider.ts +44 -5
  334. package/src/providers/retry.ts +16 -11
  335. package/src/providers/types.ts +56 -0
  336. package/src/routes/__tests__/control.test.ts +4 -3
  337. package/src/routes/control.ts +1 -1
  338. package/src/routes/route-host-client.ts +1 -1
  339. package/src/runtime/AGENTS.md +30 -12
  340. package/src/runtime/__tests__/agent-wake.test.ts +71 -6
  341. package/src/runtime/__tests__/guardian-form-registry.test.ts +242 -0
  342. package/src/runtime/access-request-helper.ts +15 -7
  343. package/src/runtime/agent-wake.ts +16 -4
  344. package/src/runtime/channel-approval-types.ts +8 -5
  345. package/src/runtime/channel-invite-transport.ts +2 -0
  346. package/src/runtime/channel-invite-transports/discord.ts +27 -0
  347. package/src/runtime/channel-readiness-service.ts +4 -4
  348. package/src/runtime/channel-reply-delivery.ts +72 -28
  349. package/src/runtime/guardian-action-service.ts +9 -5
  350. package/src/runtime/guardian-form-registry.ts +256 -0
  351. package/src/runtime/guardian-reply-router.ts +33 -14
  352. package/src/runtime/invite-instruction-generator.ts +14 -2
  353. package/src/runtime/migrations/vbundle-metadata-merge.ts +4 -7
  354. package/src/runtime/no-response.ts +13 -74
  355. package/src/runtime/routes/__tests__/acp-routes.test.ts +527 -0
  356. package/src/runtime/routes/__tests__/app-management-shared-apps.test.ts +74 -0
  357. package/src/runtime/routes/__tests__/contact-form-kinds.test.ts +22 -0
  358. package/src/runtime/routes/__tests__/contact-record-prompt-route.test.ts +303 -0
  359. package/src/runtime/routes/__tests__/monitoring-routes.test.ts +3 -4
  360. package/src/runtime/routes/__tests__/pending-interactions-route.test.ts +6 -0
  361. package/src/runtime/routes/__tests__/route-host-worker-routes.test.ts +3 -4
  362. package/src/runtime/routes/__tests__/schedule-worker-routes.test.ts +5 -6
  363. package/src/runtime/routes/acp-routes-list.test.ts +58 -0
  364. package/src/runtime/routes/acp-routes.ts +253 -47
  365. package/src/runtime/routes/app-management-routes.ts +80 -47
  366. package/src/runtime/routes/approval-routes.ts +9 -0
  367. package/src/runtime/routes/archive-utils.ts +6 -1
  368. package/src/runtime/routes/avatar-routes.ts +1 -1
  369. package/src/runtime/routes/channel-route-shared.ts +18 -20
  370. package/src/runtime/routes/contact-prompt-routes.ts +271 -105
  371. package/src/runtime/routes/conversation-routes.ts +41 -32
  372. package/src/runtime/routes/credential-prompt-routes.ts +10 -1
  373. package/src/runtime/routes/debug-bash-routes.ts +1 -0
  374. package/src/runtime/routes/guardian-expiry-sweep.ts +89 -31
  375. package/src/runtime/routes/guardian-form-routes.ts +115 -0
  376. package/src/runtime/routes/inbound-message-handler.ts +82 -29
  377. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -3
  378. package/src/runtime/routes/inbound-stages/edit-intercept.ts +6 -0
  379. package/src/runtime/routes/inbound-stages/guardian-reply-intercept.test.ts +6 -0
  380. package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +6 -1
  381. package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +50 -17
  382. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +97 -94
  383. package/src/runtime/routes/index.ts +2 -0
  384. package/src/runtime/routes/integrations/slack/channel.ts +39 -18
  385. package/src/runtime/routes/migration-routes.ts +1 -3
  386. package/src/runtime/routes/monitoring-routes.ts +0 -3
  387. package/src/runtime/routes/oauth-commands-routes.ts +18 -5
  388. package/src/runtime/routes/route-host-worker-routes.ts +1 -1
  389. package/src/runtime/routes/schedule-worker-routes.ts +1 -1
  390. package/src/runtime/tool-grant-request-helper.ts +4 -0
  391. package/src/schedule/run-script.ts +1 -0
  392. package/src/schedule/scheduler.ts +1 -1
  393. package/src/schedule/worker-control.ts +2 -2
  394. package/src/security/oauth2.ts +2 -11
  395. package/src/security/secure-keys.ts +55 -8
  396. package/src/skills/catalog-install.ts +7 -4
  397. package/src/skills/clawhub.ts +1 -0
  398. package/src/skills/inline-command-runner.ts +1 -0
  399. package/src/stt/__tests__/room-noise-floor.test.ts +108 -0
  400. package/src/stt/room-noise-floor.ts +140 -0
  401. package/src/subagent/notify.ts +49 -3
  402. package/src/tools/__tests__/tool-input-schemas.test.ts +2 -0
  403. package/src/tools/acp/spawn.ts +2 -2
  404. package/src/tools/browser/__tests__/browser-status.test.ts +9 -0
  405. package/src/tools/browser/browser-execution.ts +10 -43
  406. package/src/tools/browser/browser-manager.ts +1 -0
  407. package/src/tools/browser/browser-status-constants.ts +6 -5
  408. package/src/tools/browser/runtime-check.ts +2 -0
  409. package/src/tools/channel/react-to-message.test.ts +193 -0
  410. package/src/tools/channel/react-to-message.ts +159 -0
  411. package/src/tools/credentials/metadata-store.ts +115 -19
  412. package/src/tools/credentials/store.ts +0 -2
  413. package/src/tools/filesystem/read.ts +2 -2
  414. package/src/tools/host-filesystem/read.ts +2 -2
  415. package/src/tools/host-terminal/host-shell.test.ts +24 -0
  416. package/src/tools/host-terminal/host-shell.ts +11 -38
  417. package/src/tools/shared/filesystem/file-ops-service.ts +10 -7
  418. package/src/tools/shared/shell-output.test.ts +104 -0
  419. package/src/tools/shared/shell-output.ts +112 -39
  420. package/src/tools/skills/sandbox-runner.ts +1 -0
  421. package/src/tools/system/request-permission.test.ts +34 -0
  422. package/src/tools/system/request-permission.ts +29 -5
  423. package/src/tools/terminal/shell.test.ts +63 -0
  424. package/src/tools/terminal/shell.ts +12 -44
  425. package/src/tools/tool-input-schemas.ts +2 -0
  426. package/src/tools/tool-manifest.ts +2 -0
  427. package/src/usage/attribution.ts +12 -4
  428. package/src/util/__tests__/worker-ownership.test.ts +77 -0
  429. package/src/util/__tests__/worker-process-command.test.ts +246 -1
  430. package/src/util/bun-runtime.ts +44 -13
  431. package/src/util/bundled-asset.ts +9 -1
  432. package/src/util/disk-usage.ts +1 -0
  433. package/src/util/file-use.ts +4 -1
  434. package/src/util/guardian-form-timeouts.ts +50 -0
  435. package/src/util/image-conversion.ts +1 -0
  436. package/src/util/oauth-request-body.test.ts +59 -0
  437. package/src/util/oauth-request-body.ts +56 -5
  438. package/src/util/platform.ts +69 -73
  439. package/src/util/process-table.ts +16 -5
  440. package/src/util/read-stdin.ts +5 -0
  441. package/src/util/spawn.ts +1 -0
  442. package/src/util/sqlite3-runtime.ts +25 -5
  443. package/src/util/worker-ownership.ts +193 -0
  444. package/src/util/worker-process.ts +375 -37
  445. package/src/workspace/git-service.ts +4 -12
  446. package/src/workspace/migrations/010-app-dir-rename.ts +1 -0
@@ -0,0 +1,679 @@
1
+ /**
2
+ * Ambient camera frames sent mid-call (`attach_frame`).
3
+ *
4
+ * The behaviour under test is the parking rule: a frame dispatches nothing of
5
+ * its own. It waits on the session and rides the next spoken turn's own user
6
+ * message, so the picture and the words about it are one message, and a frame
7
+ * nobody ever speaks over leaves no trace in the conversation.
8
+ *
9
+ * The contrast with `attach_image` is deliberate and is pinned here too: a
10
+ * deliberate snap persists immediately as its own message and parks nothing
11
+ * (`live-voice-attach-image.test.ts`).
12
+ */
13
+
14
+ import { Buffer } from "node:buffer";
15
+ import { describe, expect, mock, test } from "bun:test";
16
+
17
+ import type {
18
+ VoiceTurnCallbacks,
19
+ VoiceTurnOptions,
20
+ } from "../../calls/voice-session-bridge.js";
21
+ import {
22
+ getAttachmentById,
23
+ linkAttachmentToMessage,
24
+ uploadAttachment,
25
+ } from "../../persistence/attachments-store.js";
26
+ import {
27
+ addMessage,
28
+ createConversation,
29
+ } from "../../persistence/conversation-crud.js";
30
+ import { initializeDb } from "../../persistence/db-init.js";
31
+ import type {
32
+ StreamingTranscriber,
33
+ SttStreamServerEvent,
34
+ } from "../../stt/types.js";
35
+ import {
36
+ createLiveVoiceSession,
37
+ type LiveVoiceSession,
38
+ type LiveVoiceTurnStarter,
39
+ } from "../live-voice-session.js";
40
+ import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
41
+ import type {
42
+ LiveVoiceTtsOptions,
43
+ LiveVoiceTtsResult,
44
+ } from "../live-voice-tts.js";
45
+ import {
46
+ createLiveVoiceServerFrameSequencer,
47
+ type LiveVoiceClientStartFrame,
48
+ type LiveVoiceServerFrame,
49
+ validateLiveVoiceClientFrame,
50
+ } from "../protocol.js";
51
+
52
+ await initializeDb();
53
+
54
+ const IMAGE_BASE64 =
55
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk";
56
+
57
+ const START_FRAME = {
58
+ type: "start",
59
+ conversationId: "conversation-123",
60
+ audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
61
+ turnDetection: "server_vad",
62
+ } as const satisfies LiveVoiceClientStartFrame;
63
+
64
+ /** A real attachment row, so the park-time existence check has one to find. */
65
+ async function uploadFrame(): Promise<string> {
66
+ const attachment = await uploadAttachment(
67
+ "frame.png",
68
+ "image/png",
69
+ IMAGE_BASE64,
70
+ );
71
+ return attachment.id;
72
+ }
73
+
74
+ /** True while the attachment's row is still in the store. */
75
+ function frameStored(attachmentId: string): boolean {
76
+ return getAttachmentById(attachmentId) !== null;
77
+ }
78
+
79
+ function loudPcmChunk(amplitude: number, sampleCount = 240): Uint8Array {
80
+ const buffer = Buffer.alloc(sampleCount * 2);
81
+ for (let index = 0; index < sampleCount; index += 1) {
82
+ buffer.writeInt16LE(amplitude, index * 2);
83
+ }
84
+ return new Uint8Array(buffer);
85
+ }
86
+
87
+ /** One fresh transcriber per utterance; `stop()` flushes its transcript. */
88
+ class FakeStreamingTranscriber implements StreamingTranscriber {
89
+ readonly providerId = "deepgram" as const;
90
+ readonly boundaryId = "daemon-streaming" as const;
91
+ private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
92
+
93
+ constructor(private readonly transcript: string) {}
94
+
95
+ async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
96
+ this.onEvent = onEvent;
97
+ }
98
+
99
+ sendAudio(): void {
100
+ this.onEvent?.({ type: "partial", text: "wha" });
101
+ }
102
+
103
+ stop(): void {
104
+ this.onEvent?.({ type: "final", text: this.transcript });
105
+ this.onEvent?.({ type: "closed" });
106
+ }
107
+ }
108
+
109
+ function makeMessageComplete(): Parameters<
110
+ NonNullable<VoiceTurnCallbacks["message_complete"]>
111
+ >[0] {
112
+ return {
113
+ type: "message_complete",
114
+ conversationId: "conversation-123",
115
+ messageId: "assistant-message-123",
116
+ };
117
+ }
118
+
119
+ function makeTtsResult(text: string): LiveVoiceTtsResult {
120
+ return {
121
+ provider: "fish-audio",
122
+ contentType: "audio/pcm",
123
+ sampleRate: 24_000,
124
+ chunks: 1,
125
+ bytes: Buffer.byteLength(text),
126
+ };
127
+ }
128
+
129
+ function createSessionHarness(startVoiceTurn: LiveVoiceTurnStarter) {
130
+ const sequencer = createLiveVoiceServerFrameSequencer();
131
+ const frames: LiveVoiceServerFrame[] = [];
132
+ const context: LiveVoiceSessionFactoryContext = {
133
+ sessionId: "session-123",
134
+ startFrame: START_FRAME,
135
+ sendFrame: mock(async (payload) => {
136
+ const frame = sequencer.next(payload);
137
+ frames.push(frame);
138
+ return frame;
139
+ }),
140
+ };
141
+
142
+ let utterances = 0;
143
+ const session = createLiveVoiceSession(context, {
144
+ // Credential-free harness: every leg is injected, so skip the preflight.
145
+ resolveCredentialReadiness: null,
146
+ // One discrete mic chunk per utterance; keep the adaptive playback
147
+ // classifier out of the cycle timing.
148
+ echoBargeInMargin: 1,
149
+ resolveTranscriber: mock(async () => {
150
+ utterances += 1;
151
+ return new FakeStreamingTranscriber(`utterance ${utterances}`);
152
+ }),
153
+ startVoiceTurn,
154
+ streamTtsAudio: mock(async (options: LiveVoiceTtsOptions) =>
155
+ makeTtsResult(options.text),
156
+ ),
157
+ createTurnId: () => `live-turn-${utterances}`,
158
+ emitMetrics: false,
159
+ });
160
+
161
+ return { frames, session };
162
+ }
163
+
164
+ async function waitFor(
165
+ predicate: () => boolean,
166
+ message: string,
167
+ ): Promise<void> {
168
+ for (let attempt = 0; attempt < 200; attempt += 1) {
169
+ if (predicate()) {
170
+ return;
171
+ }
172
+ await new Promise((resolve) => setTimeout(resolve, 5));
173
+ }
174
+ throw new Error(message);
175
+ }
176
+
177
+ /** Drive one utterance through to its dispatched assistant turn. */
178
+ async function speakAndRelease(
179
+ session: LiveVoiceSession,
180
+ seen: () => number,
181
+ amplitude: number,
182
+ ): Promise<void> {
183
+ const before = seen();
184
+ await session.handleBinaryAudio(loudPcmChunk(amplitude));
185
+ await session.handleClientFrame({ type: "ptt_release" });
186
+ await waitFor(
187
+ () => seen() > before,
188
+ "Timed out waiting for the live-voice turn to dispatch",
189
+ );
190
+ }
191
+
192
+ function parkedFrameId(session: LiveVoiceSession): string | null {
193
+ return (session as unknown as { pendingTurnAttachmentId: string | null })
194
+ .pendingTurnAttachmentId;
195
+ }
196
+
197
+ describe("live-voice attach_frame frame", () => {
198
+ test("accepts a well-formed frame", () => {
199
+ const result = validateLiveVoiceClientFrame({
200
+ type: "attach_frame",
201
+ attachmentId: "att-1",
202
+ });
203
+ expect(result).toEqual({
204
+ ok: true,
205
+ frame: { type: "attach_frame", attachmentId: "att-1" },
206
+ });
207
+ });
208
+
209
+ test("accepts a null attachmentId, which unparks", () => {
210
+ const result = validateLiveVoiceClientFrame({
211
+ type: "attach_frame",
212
+ attachmentId: null,
213
+ });
214
+ expect(result).toEqual({
215
+ ok: true,
216
+ frame: { type: "attach_frame", attachmentId: null },
217
+ });
218
+ });
219
+
220
+ test("rejects a missing attachmentId, naming the frame", () => {
221
+ // A missing field is not an unpark. A client that means to clear the slot
222
+ // says so with an explicit null; an absent field is a malformed frame.
223
+ const result = validateLiveVoiceClientFrame({ type: "attach_frame" });
224
+ expect(result.ok).toBe(false);
225
+ if (!result.ok) {
226
+ expect(result.error.code).toBe("missing_required_field");
227
+ expect(result.error.field).toBe("attachmentId");
228
+ expect(result.error.frameType).toBe("attach_frame");
229
+ }
230
+ });
231
+
232
+ test("rejects an empty attachmentId", () => {
233
+ const result = validateLiveVoiceClientFrame({
234
+ type: "attach_frame",
235
+ attachmentId: "",
236
+ });
237
+ expect(result.ok).toBe(false);
238
+ if (!result.ok) {
239
+ expect(result.error.code).toBe("invalid_field");
240
+ expect(result.error.frameType).toBe("attach_frame");
241
+ }
242
+ });
243
+ });
244
+
245
+ describe("live-voice camera frames parked for the next turn", () => {
246
+ test("a parked frame rides the next turn's user message", async () => {
247
+ const attachmentId = await uploadFrame();
248
+ const turns: VoiceTurnOptions[] = [];
249
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
250
+ async (options: VoiceTurnOptions) => {
251
+ turns.push(options);
252
+ options.callbacks?.message_complete?.(makeMessageComplete());
253
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
254
+ },
255
+ );
256
+ const { session } = createSessionHarness(startVoiceTurn);
257
+ await session.start();
258
+
259
+ await session.handleClientFrame({ type: "attach_frame", attachmentId });
260
+ // Nothing is dispatched for the frame itself; it waits for words.
261
+ expect(turns).toHaveLength(0);
262
+ expect(parkedFrameId(session)).toBe(attachmentId);
263
+
264
+ await speakAndRelease(session, () => turns.length, 8_000);
265
+
266
+ expect(turns[0]?.content).toBe("utterance 1");
267
+ expect(turns[0]?.attachments).toEqual([attachmentId]);
268
+ // Drained by the turn that took it, so it cannot ride a second one.
269
+ expect(parkedFrameId(session)).toBeNull();
270
+
271
+ await session.close("websocket_close");
272
+ });
273
+
274
+ test("the newest frame replaces the one before it", async () => {
275
+ const stale = await uploadFrame();
276
+ const current = await uploadFrame();
277
+ const turns: VoiceTurnOptions[] = [];
278
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
279
+ async (options: VoiceTurnOptions) => {
280
+ turns.push(options);
281
+ options.callbacks?.message_complete?.(makeMessageComplete());
282
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
283
+ },
284
+ );
285
+ const { session } = createSessionHarness(startVoiceTurn);
286
+ await session.start();
287
+
288
+ await session.handleClientFrame({
289
+ type: "attach_frame",
290
+ attachmentId: stale,
291
+ });
292
+ await session.handleClientFrame({
293
+ type: "attach_frame",
294
+ attachmentId: current,
295
+ });
296
+
297
+ // A client shooting the camera on a timer parks one of these every few
298
+ // seconds. Nothing else ever collects them, so the displaced one is given
299
+ // up here or its row and bytes stay for good.
300
+ expect(frameStored(stale)).toBe(false);
301
+ expect(frameStored(current)).toBe(true);
302
+
303
+ await speakAndRelease(session, () => turns.length, 8_000);
304
+
305
+ expect(turns[0]?.attachments).toEqual([current]);
306
+
307
+ await session.close("websocket_close");
308
+ });
309
+
310
+ test("displacing a frame that already rode a message leaves it alone", async () => {
311
+ // The reclaim is link-aware, which is what makes it safe to run on an id
312
+ // the client may have sent before: a frame that reached a message is that
313
+ // message's, and re-sending its id must not delete it out from under the
314
+ // transcript.
315
+ const conversation = createConversation("Live voice frame reuse");
316
+ const message = await addMessage(conversation.id, "user", "what is this");
317
+ const sent = await uploadFrame();
318
+ linkAttachmentToMessage(message.id, sent, 0);
319
+ const replacement = await uploadFrame();
320
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
321
+ async (options: VoiceTurnOptions) => {
322
+ options.callbacks?.message_complete?.(makeMessageComplete());
323
+ return { turnId: "bridge-turn-1", abort: mock() };
324
+ },
325
+ );
326
+ const { session } = createSessionHarness(startVoiceTurn);
327
+ await session.start();
328
+
329
+ await session.handleClientFrame({
330
+ type: "attach_frame",
331
+ attachmentId: sent,
332
+ });
333
+ await session.handleClientFrame({
334
+ type: "attach_frame",
335
+ attachmentId: replacement,
336
+ });
337
+
338
+ expect(frameStored(sent)).toBe(true);
339
+ expect(parkedFrameId(session)).toBe(replacement);
340
+
341
+ await session.close("websocket_close");
342
+ });
343
+
344
+ test("a frame that arrives mid-turn rides the turn after it", async () => {
345
+ const first = await uploadFrame();
346
+ const second = await uploadFrame();
347
+ const turns: VoiceTurnOptions[] = [];
348
+ const sessionRef: { current: LiveVoiceSession | null } = { current: null };
349
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
350
+ async (options: VoiceTurnOptions) => {
351
+ turns.push(options);
352
+ if (turns.length === 1) {
353
+ // The camera keeps shooting while the assistant answers. This frame
354
+ // belongs to whatever the user says next, not to the turn in flight.
355
+ await sessionRef.current?.handleClientFrame({
356
+ type: "attach_frame",
357
+ attachmentId: second,
358
+ });
359
+ }
360
+ options.callbacks?.message_complete?.(makeMessageComplete());
361
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
362
+ },
363
+ );
364
+ const { session } = createSessionHarness(startVoiceTurn);
365
+ sessionRef.current = session;
366
+ await session.start();
367
+
368
+ await session.handleClientFrame({
369
+ type: "attach_frame",
370
+ attachmentId: first,
371
+ });
372
+ await speakAndRelease(session, () => turns.length, 8_000);
373
+ await speakAndRelease(session, () => turns.length, 9_000);
374
+
375
+ expect(turns[0]?.attachments).toEqual([first]);
376
+ expect(turns[1]?.attachments).toEqual([second]);
377
+
378
+ await session.close("websocket_close");
379
+ });
380
+
381
+ test("unparking clears the slot and gives the frame up", async () => {
382
+ // The client's viewfinder closed. Nothing should still be staged for a
383
+ // turn that would show a view the user can no longer see.
384
+ const attachmentId = await uploadFrame();
385
+ const turns: VoiceTurnOptions[] = [];
386
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
387
+ async (options: VoiceTurnOptions) => {
388
+ turns.push(options);
389
+ options.callbacks?.message_complete?.(makeMessageComplete());
390
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
391
+ },
392
+ );
393
+ const { frames, session } = createSessionHarness(startVoiceTurn);
394
+ await session.start();
395
+
396
+ await session.handleClientFrame({ type: "attach_frame", attachmentId });
397
+ expect(parkedFrameId(session)).toBe(attachmentId);
398
+ const before = frames.length;
399
+
400
+ await session.handleClientFrame({
401
+ type: "attach_frame",
402
+ attachmentId: null,
403
+ });
404
+
405
+ expect(parkedFrameId(session)).toBeNull();
406
+ expect(frameStored(attachmentId)).toBe(false);
407
+ // Nothing is answered for an unpark: it is the client tidying up after
408
+ // itself, not a request that can fail.
409
+ expect(frames.slice(before)).toEqual([]);
410
+
411
+ await speakAndRelease(session, () => turns.length, 8_000);
412
+ expect(turns[0]?.attachments).toBeUndefined();
413
+
414
+ await session.close("websocket_close");
415
+ });
416
+
417
+ test("unparking an empty slot does nothing", async () => {
418
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
419
+ async (options: VoiceTurnOptions) => {
420
+ options.callbacks?.message_complete?.(makeMessageComplete());
421
+ return { turnId: "bridge-turn-1", abort: mock() };
422
+ },
423
+ );
424
+ const { frames, session } = createSessionHarness(startVoiceTurn);
425
+ await session.start();
426
+ const before = frames.length;
427
+
428
+ await session.handleClientFrame({
429
+ type: "attach_frame",
430
+ attachmentId: null,
431
+ });
432
+
433
+ expect(parkedFrameId(session)).toBeNull();
434
+ expect(frames.slice(before)).toEqual([]);
435
+
436
+ await session.close("websocket_close");
437
+ });
438
+
439
+ test("unparking leaves a frame that already rode a message alone", async () => {
440
+ // Same link-awareness the displacement path relies on: an unpark arriving
441
+ // after the turn drained the slot finds it empty, and the frame the
442
+ // transcript is showing is not this call's to collect.
443
+ const conversation = createConversation("Live voice frame unpark");
444
+ const message = await addMessage(conversation.id, "user", "what is this");
445
+ const sent = await uploadFrame();
446
+ linkAttachmentToMessage(message.id, sent, 0);
447
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
448
+ async (options: VoiceTurnOptions) => {
449
+ options.callbacks?.message_complete?.(makeMessageComplete());
450
+ return { turnId: "bridge-turn-1", abort: mock() };
451
+ },
452
+ );
453
+ const { session } = createSessionHarness(startVoiceTurn);
454
+ await session.start();
455
+
456
+ await session.handleClientFrame({
457
+ type: "attach_frame",
458
+ attachmentId: sent,
459
+ });
460
+ await session.handleClientFrame({
461
+ type: "attach_frame",
462
+ attachmentId: null,
463
+ });
464
+
465
+ expect(parkedFrameId(session)).toBeNull();
466
+ expect(frameStored(sent)).toBe(true);
467
+
468
+ await session.close("websocket_close");
469
+ });
470
+
471
+ test("ending the session drops the parked frame", async () => {
472
+ const attachmentId = await uploadFrame();
473
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
474
+ async (options: VoiceTurnOptions) => {
475
+ options.callbacks?.message_complete?.(makeMessageComplete());
476
+ return { turnId: "bridge-turn-1", abort: mock() };
477
+ },
478
+ );
479
+ const { session } = createSessionHarness(startVoiceTurn);
480
+ await session.start();
481
+
482
+ await session.handleClientFrame({ type: "attach_frame", attachmentId });
483
+ expect(parkedFrameId(session)).toBe(attachmentId);
484
+
485
+ await session.close("websocket_close");
486
+
487
+ // A frame is only good for the call it was shot on, so the staged row
488
+ // goes with the call rather than outliving it.
489
+ expect(parkedFrameId(session)).toBeNull();
490
+ expect(frameStored(attachmentId)).toBe(false);
491
+ });
492
+
493
+ test("a rolled-back turn hands the frame back for the replay", async () => {
494
+ // A turn that never reached the bridge (and a speculative one the hold
495
+ // verdict unwinds) must not take the frame down with it: the utterance is
496
+ // about to be sent again, and the user is still holding the thing up.
497
+ // The other half of this, that the rollback leaves the attachment itself
498
+ // alive, is `voice-session-bridge.test.ts`'s discard suite.
499
+ const attachmentId = await uploadFrame();
500
+ const turns: VoiceTurnOptions[] = [];
501
+ let dispatches = 0;
502
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
503
+ async (options: VoiceTurnOptions) => {
504
+ dispatches += 1;
505
+ if (dispatches === 1) {
506
+ throw new Error("bridge unavailable");
507
+ }
508
+ turns.push(options);
509
+ options.callbacks?.message_complete?.(makeMessageComplete());
510
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
511
+ },
512
+ );
513
+ const { session } = createSessionHarness(startVoiceTurn);
514
+ await session.start();
515
+
516
+ await session.handleClientFrame({ type: "attach_frame", attachmentId });
517
+ await speakAndRelease(session, () => dispatches, 8_000);
518
+
519
+ // Claimed by the failed turn, then handed straight back.
520
+ expect(parkedFrameId(session)).toBe(attachmentId);
521
+
522
+ await speakAndRelease(session, () => dispatches, 9_000);
523
+
524
+ expect(turns[0]?.attachments).toEqual([attachmentId]);
525
+
526
+ await session.close("websocket_close");
527
+ });
528
+
529
+ test("a rollback keeps the newer frame and gives up the one it replaced", async () => {
530
+ // The camera kept shooting while the doomed turn was in flight, so the
531
+ // frame coming back off it is already out of date. Latest still wins, and
532
+ // the one that loses is nobody's to send: it goes.
533
+ const claimed = await uploadFrame();
534
+ const newer = await uploadFrame();
535
+ const turns: VoiceTurnOptions[] = [];
536
+ let dispatches = 0;
537
+ const sessionRef: { current: LiveVoiceSession | null } = { current: null };
538
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
539
+ async (options: VoiceTurnOptions) => {
540
+ dispatches += 1;
541
+ if (dispatches === 1) {
542
+ await sessionRef.current?.handleClientFrame({
543
+ type: "attach_frame",
544
+ attachmentId: newer,
545
+ });
546
+ throw new Error("bridge unavailable");
547
+ }
548
+ turns.push(options);
549
+ options.callbacks?.message_complete?.(makeMessageComplete());
550
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
551
+ },
552
+ );
553
+ const { session } = createSessionHarness(startVoiceTurn);
554
+ sessionRef.current = session;
555
+ await session.start();
556
+
557
+ await session.handleClientFrame({
558
+ type: "attach_frame",
559
+ attachmentId: claimed,
560
+ });
561
+ await speakAndRelease(session, () => dispatches, 8_000);
562
+
563
+ expect(parkedFrameId(session)).toBe(newer);
564
+ expect(frameStored(claimed)).toBe(false);
565
+
566
+ await speakAndRelease(session, () => dispatches, 9_000);
567
+
568
+ expect(turns[0]?.attachments).toEqual([newer]);
569
+
570
+ await session.close("websocket_close");
571
+ });
572
+
573
+ test("a retransmitted id is the same frame, not a newer one", async () => {
574
+ // A client that re-sends the id it already sent, while the turn holding it
575
+ // is in flight, has parked the SAME frame. Reading that as a displacement
576
+ // would collect the very frame the slot is pointing at, and the replay
577
+ // would speak without the picture with nothing said.
578
+ const attachmentId = await uploadFrame();
579
+ const turns: VoiceTurnOptions[] = [];
580
+ let dispatches = 0;
581
+ const sessionRef: { current: LiveVoiceSession | null } = { current: null };
582
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
583
+ async (options: VoiceTurnOptions) => {
584
+ dispatches += 1;
585
+ if (dispatches === 1) {
586
+ await sessionRef.current?.handleClientFrame({
587
+ type: "attach_frame",
588
+ attachmentId,
589
+ });
590
+ throw new Error("bridge unavailable");
591
+ }
592
+ turns.push(options);
593
+ options.callbacks?.message_complete?.(makeMessageComplete());
594
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
595
+ },
596
+ );
597
+ const { session } = createSessionHarness(startVoiceTurn);
598
+ sessionRef.current = session;
599
+ await session.start();
600
+
601
+ await session.handleClientFrame({ type: "attach_frame", attachmentId });
602
+ await speakAndRelease(session, () => dispatches, 8_000);
603
+
604
+ expect(parkedFrameId(session)).toBe(attachmentId);
605
+ expect(frameStored(attachmentId)).toBe(true);
606
+
607
+ await speakAndRelease(session, () => dispatches, 9_000);
608
+
609
+ expect(turns[0]?.attachments).toEqual([attachmentId]);
610
+
611
+ await session.close("websocket_close");
612
+ });
613
+
614
+ test("an unknown attachment id is refused and nothing is parked", async () => {
615
+ const turns: VoiceTurnOptions[] = [];
616
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
617
+ async (options: VoiceTurnOptions) => {
618
+ turns.push(options);
619
+ options.callbacks?.message_complete?.(makeMessageComplete());
620
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
621
+ },
622
+ );
623
+ const { frames, session } = createSessionHarness(startVoiceTurn);
624
+ await session.start();
625
+ const before = frames.length;
626
+
627
+ await session.handleClientFrame({
628
+ type: "attach_frame",
629
+ attachmentId: "att-missing",
630
+ });
631
+ await waitFor(
632
+ () => frames.length > before,
633
+ "Timed out waiting for the rejected camera frame's error",
634
+ );
635
+
636
+ // Attributed to the frame it is about, so the client can retract the
637
+ // preview it already showed instead of filing this with the transient
638
+ // transcriber and TTS blips that share `recoverable`.
639
+ expect(
640
+ frames.slice(before).find((frame) => frame.type === "error"),
641
+ ).toMatchObject({
642
+ type: "error",
643
+ frameType: "attach_frame",
644
+ recoverable: true,
645
+ });
646
+ expect(parkedFrameId(session)).toBeNull();
647
+
648
+ await speakAndRelease(session, () => turns.length, 8_000);
649
+ expect(turns[0]?.attachments).toBeUndefined();
650
+
651
+ await session.close("websocket_close");
652
+ });
653
+
654
+ test("attach_image parks nothing", async () => {
655
+ // A deliberate snap persists as its own message; the spoken turn that
656
+ // follows carries no attachments and reaches it through history.
657
+ const turns: VoiceTurnOptions[] = [];
658
+ const startVoiceTurn: LiveVoiceTurnStarter = mock(
659
+ async (options: VoiceTurnOptions) => {
660
+ turns.push(options);
661
+ options.callbacks?.message_complete?.(makeMessageComplete());
662
+ return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
663
+ },
664
+ );
665
+ const { session } = createSessionHarness(startVoiceTurn);
666
+ await session.start();
667
+
668
+ await session.handleClientFrame({
669
+ type: "attach_image",
670
+ attachmentId: "att-missing",
671
+ });
672
+ expect(parkedFrameId(session)).toBeNull();
673
+
674
+ await speakAndRelease(session, () => turns.length, 8_000);
675
+ expect(turns[0]?.attachments).toBeUndefined();
676
+
677
+ await session.close("websocket_close");
678
+ });
679
+ });
@@ -1,11 +1,14 @@
1
1
  /**
2
2
  * Photos taken mid-call (the voice room's camera).
3
3
  *
4
- * The behaviour under test is the parking rule: an `attach_image` frame does
5
- * not dispatch anything, it waits for the next turn and rides that turn's own
6
- * user message. That is what makes a bare "what's this?" resolve against the
7
- * picture instead of producing one turn about the image racing another about
8
- * the words.
4
+ * The behaviour under test is that an `attach_image` frame stands alone: it
5
+ * persists the photo as its own user message the moment it arrives and
6
+ * dispatches no turn, so the spoken turn that follows carries no attachments
7
+ * of its own and reaches the picture through conversation history. That is
8
+ * what makes shutter-then-speak and speak-then-shutter answer the same way.
9
+ *
10
+ * The parking rule belongs to `attach_frame` instead
11
+ * (`live-voice-attach-frame.test.ts`), which carries ambient camera frames.
9
12
  */
10
13
 
11
14
  import { describe, expect, mock, test } from "bun:test";