@vellumai/assistant 0.11.7 → 0.11.8-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/ARCHITECTURE.md +10 -10
  2. package/Dockerfile +1 -0
  3. package/docs/architecture/security.md +1 -1
  4. package/docs/error-handling.md +1 -2
  5. package/docs/guardian-request-flow.md +51 -5
  6. package/docs/trusted-contact-access.md +13 -12
  7. package/knip.json +1 -0
  8. package/node_modules/@vellumai/avatar-catalog/package.json +19 -0
  9. package/{src/avatar/character-components.ts → node_modules/@vellumai/avatar-catalog/src/catalog.ts} +12 -44
  10. package/node_modules/@vellumai/avatar-catalog/src/colors.ts +19 -0
  11. package/node_modules/@vellumai/avatar-catalog/src/index.ts +25 -0
  12. package/node_modules/@vellumai/avatar-catalog/src/types.ts +47 -0
  13. package/node_modules/@vellumai/avatar-catalog/tsconfig.json +20 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
  18. package/node_modules/@vellumai/credential-storage/src/static-credentials.ts +43 -12
  19. package/node_modules/@vellumai/environments/package.json +2 -1
  20. package/node_modules/@vellumai/environments/src/__tests__/test-path-guard.test.ts +89 -0
  21. package/node_modules/@vellumai/environments/src/test-path-guard.ts +113 -0
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
  26. package/node_modules/@vellumai/gateway-client/src/__tests__/guardian-request-contract.test.ts +8 -18
  27. package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +46 -0
  28. package/node_modules/@vellumai/gateway-client/src/__tests__/invite-contract.test.ts +1 -0
  29. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +46 -0
  30. package/node_modules/@vellumai/gateway-client/src/guardian-request-contract.ts +62 -21
  31. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +14 -0
  32. package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +53 -0
  33. package/node_modules/@vellumai/gateway-client/src/index.ts +11 -5
  34. package/node_modules/@vellumai/gateway-client/src/invite-contract.ts +1 -0
  35. package/node_modules/@vellumai/service-contracts/package.json +1 -0
  36. package/node_modules/@vellumai/service-contracts/src/channels.ts +7 -0
  37. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  38. package/node_modules/@vellumai/service-contracts/src/no-response.ts +84 -0
  39. package/openapi.yaml +481 -9
  40. package/package.json +3 -1
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/src/__tests__/agent-loop-callsite-precedence.test.ts +16 -0
  43. package/src/__tests__/agent-loop-compacted-history-transform.test.ts +304 -0
  44. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +3 -0
  45. package/src/__tests__/agent-wake-override-profile.test.ts +3 -0
  46. package/src/__tests__/anthropic-provider.test.ts +56 -0
  47. package/src/__tests__/approval-cascade.test.ts +0 -13
  48. package/src/__tests__/attachments-store.test.ts +117 -6
  49. package/src/__tests__/attachments.test.ts +122 -0
  50. package/src/__tests__/bundled-asset.test.ts +22 -1
  51. package/src/__tests__/channel-approval.test.ts +14 -7
  52. package/src/__tests__/channel-approvals.test.ts +3 -3
  53. package/src/__tests__/channel-delivery-store.test.ts +58 -0
  54. package/src/__tests__/channel-inbound-disk-pressure.test.ts +1 -0
  55. package/src/__tests__/channel-invite-transport.test.ts +1 -0
  56. package/src/__tests__/channel-readiness-slack-remote.test.ts +3 -3
  57. package/src/__tests__/compaction-events.test.ts +0 -12
  58. package/src/__tests__/compactor-call-site-logging.test.ts +7 -1
  59. package/src/__tests__/compactor-retained-image-validation.test.ts +15 -0
  60. package/src/__tests__/config-schema.test.ts +35 -0
  61. package/src/__tests__/contacts-write.test.ts +141 -3
  62. package/src/__tests__/conversation-abort-tool-results.test.ts +0 -13
  63. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +0 -12
  64. package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -13
  65. package/src/__tests__/conversation-agent-loop.test.ts +95 -13
  66. package/src/__tests__/conversation-confirmation-signals.test.ts +0 -13
  67. package/src/__tests__/conversation-load-history-repair.test.ts +418 -0
  68. package/src/__tests__/conversation-media-retry.test.ts +33 -2
  69. package/src/__tests__/conversation-pairing.test.ts +54 -0
  70. package/src/__tests__/conversation-process-callsite.test.ts +0 -12
  71. package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -13
  72. package/src/__tests__/conversation-queue.test.ts +4 -13
  73. package/src/__tests__/conversation-retry-route.test.ts +5 -0
  74. package/src/__tests__/conversation-routes-disk-view.test.ts +1 -1
  75. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +1 -1
  76. package/src/__tests__/conversation-routes-hidden-queue.test.ts +1 -1
  77. package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -1
  78. package/src/__tests__/conversation-sight-frames.test.ts +615 -0
  79. package/src/__tests__/conversation-slash-queue.test.ts +0 -13
  80. package/src/__tests__/conversation-slash-unknown.test.ts +0 -13
  81. package/src/__tests__/conversation-speed-override.test.ts +0 -12
  82. package/src/__tests__/conversation-store-ephemeral.test.ts +0 -12
  83. package/src/__tests__/conversation-summarize-route.test.ts +5 -0
  84. package/src/__tests__/conversation-summarize-up-to.test.ts +102 -0
  85. package/src/__tests__/conversation-wait-for-idle.test.ts +0 -12
  86. package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -5
  87. package/src/__tests__/conversation-workspace-injection.test.ts +0 -17
  88. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -17
  89. package/src/__tests__/credential-health-service.test.ts +45 -0
  90. package/src/__tests__/credential-record-retire.test.ts +256 -0
  91. package/src/__tests__/credential-record-write-through.test.ts +8 -4
  92. package/src/__tests__/db-conversation-fork-lineage-migration.test.ts +0 -4
  93. package/src/__tests__/db-conversation-inference-profile-migration.test.ts +0 -4
  94. package/src/__tests__/db-llm-request-log-provider-migration.test.ts +0 -8
  95. package/src/__tests__/delete-propagation.test.ts +121 -3
  96. package/src/__tests__/evict-conversations-for-reload.test.ts +46 -6
  97. package/src/__tests__/file-ops-service.test.ts +8 -3
  98. package/src/__tests__/gemini-provider.test.ts +64 -0
  99. package/src/__tests__/guardian-action-service-scope.test.ts +118 -0
  100. package/src/__tests__/guardian-card-withdrawal.test.ts +299 -5
  101. package/src/__tests__/guardian-expiry-notifier.test.ts +117 -13
  102. package/src/__tests__/guardian-gateway-sim.test.ts +119 -0
  103. package/src/__tests__/guardian-gateway-sim.ts +56 -20
  104. package/src/__tests__/guardian-reply-router-answer-mode.test.ts +6 -0
  105. package/src/__tests__/guardian-request-status-sync.test.ts +162 -0
  106. package/src/__tests__/guardian-wait-controller.test.ts +6 -0
  107. package/src/__tests__/headless-browser-mode.test.ts +27 -0
  108. package/src/__tests__/helpers/mock-conversation.ts +2 -0
  109. package/src/__tests__/list-messages-system-card.test.ts +70 -0
  110. package/src/__tests__/llm-catalog-parity.test.ts +2 -0
  111. package/src/__tests__/managed-profile-guard.test.ts +2 -2
  112. package/src/__tests__/media-resolve-image-validation.test.ts +106 -0
  113. package/src/__tests__/media-stream-server-integration.test.ts +6 -0
  114. package/src/__tests__/non-member-access-request.test.ts +12 -0
  115. package/src/__tests__/notification-discord-adapter.test.ts +76 -4
  116. package/src/__tests__/oauth-commands-routes.test.ts +21 -0
  117. package/src/__tests__/openai-provider.test.ts +63 -0
  118. package/src/__tests__/outbound-slack-persistence.test.ts +114 -2
  119. package/src/__tests__/persist-unsendable-image-downscale.test.ts +63 -1
  120. package/src/__tests__/persist-unsendable-image.test.ts +19 -0
  121. package/src/__tests__/plugin-app-serve-routes.test.ts +2 -2
  122. package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -0
  123. package/src/__tests__/process-message-background-slack.test.ts +1 -1
  124. package/src/__tests__/process-message-display-content.test.ts +1 -1
  125. package/src/__tests__/processing-flag-persist-failure.test.ts +0 -5
  126. package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +0 -17
  127. package/src/__tests__/qdrant-manager.test.ts +6 -0
  128. package/src/__tests__/reaction-persistence.test.ts +58 -62
  129. package/src/__tests__/retry-thinking-adaptive-only.test.ts +67 -0
  130. package/src/__tests__/server-history-render.test.ts +22 -0
  131. package/src/__tests__/sight-frame-retention.test.ts +362 -0
  132. package/src/__tests__/slack-reaction-approvals.test.ts +44 -24
  133. package/src/__tests__/slack-reaction-guardian-approval.test.ts +29 -1
  134. package/src/__tests__/subagent-fork-notifications.test.ts +2 -0
  135. package/src/__tests__/subagent-manager-notify.test.ts +2 -0
  136. package/src/__tests__/subagent-notify-parent.test.ts +2 -0
  137. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +85 -0
  138. package/src/__tests__/subagent-tool-gate-mode.test.ts +4 -4
  139. package/src/__tests__/terminal-tools.test.ts +9 -9
  140. package/src/__tests__/tool-result-spool.test.ts +5 -8
  141. package/src/__tests__/usage-attribution.test.ts +26 -0
  142. package/src/acp/__tests__/acp-auth-marker-store.test.ts +191 -0
  143. package/src/acp/__tests__/acp-claude-oauth.test.ts +262 -0
  144. package/src/acp/__tests__/acp-connect-card-state.test.ts +17 -0
  145. package/src/acp/__tests__/helpers/acp-history-db.ts +10 -3
  146. package/src/acp/acp-auth-marker-store.ts +196 -0
  147. package/src/acp/acp-claude-oauth.ts +154 -4
  148. package/src/acp/acp-connect-card-state.ts +49 -8
  149. package/src/acp/agent-process.ts +1 -0
  150. package/src/acp/auto-install.ts +1 -0
  151. package/src/acp/client-handler.ts +1 -0
  152. package/src/acp/prepare-agent-env.ts +79 -1
  153. package/src/acp/resolve-agent.ts +12 -0
  154. package/src/acp/session-manager.test.ts +183 -0
  155. package/src/acp/session-manager.ts +110 -6
  156. package/src/acp/types.ts +17 -0
  157. package/src/agent/attachments.ts +7 -2
  158. package/src/agent/loop.ts +59 -1
  159. package/src/api/events/contact-form-closed.ts +32 -0
  160. package/src/api/events/contact-record-request.ts +63 -0
  161. package/src/api/events/contact-request.ts +10 -3
  162. package/src/api/index.ts +20 -0
  163. package/src/api/responses/conversation-message.ts +20 -0
  164. package/src/api/responses/home.ts +101 -0
  165. package/src/approvals/AGENTS.md +1 -1
  166. package/src/approvals/guardian-card-withdrawal.ts +127 -30
  167. package/src/approvals/guardian-decision-primitive.ts +9 -0
  168. package/src/approvals/guardian-expiry-notifier.ts +22 -16
  169. package/src/approvals/guardian-request-status-sync.ts +98 -0
  170. package/src/avatar/svg-compositor.ts +1 -1
  171. package/src/avatar/traits-png-sync.ts +1 -1
  172. package/src/bundler/app-compiler.ts +1 -0
  173. package/src/bundler/compiler-tools.ts +1 -0
  174. package/src/bundler/package-resolver.ts +1 -0
  175. package/src/calls/__tests__/voice-session-bridge.test.ts +183 -3
  176. package/src/calls/guardian-action-sweep.ts +6 -1
  177. package/src/calls/voice-session-bridge.ts +46 -1
  178. package/src/channels/__tests__/gateway-guardian-requests.test.ts +13 -18
  179. package/src/channels/gateway-guardian-requests.ts +37 -18
  180. package/src/cli/__tests__/contacts-record-prompt-cli.test.ts +329 -0
  181. package/src/cli/commands/__tests__/browser.test.ts +65 -1
  182. package/src/cli/commands/browser.ts +30 -10
  183. package/src/cli/commands/contacts.help.ts +173 -11
  184. package/src/cli/commands/contacts.ts +343 -4
  185. package/src/cli/commands/db/refresh.ts +1 -0
  186. package/src/cli/commands/oauth/index.help.ts +6 -3
  187. package/src/cli/commands/oauth/request.test.ts +37 -0
  188. package/src/cli/commands/oauth/request.ts +7 -6
  189. package/src/cli/lib/bundled-marketplace.json +1 -1
  190. package/src/cli/lib/install-from-github.ts +20 -19
  191. package/src/cli/lib/install-plugin-dependencies.ts +3 -5
  192. package/src/cli/lib/merge-plugin-tree.ts +1 -0
  193. package/src/cli/lib/publish-plugin.ts +4 -1
  194. package/src/config/__tests__/balanced-model-experiment.test.ts +63 -31
  195. package/src/config/__tests__/default-profile-catalog.test.ts +2 -2
  196. package/src/config/balanced-model-experiment.ts +15 -9
  197. package/src/config/bundled-skills/contacts/SKILL.md +61 -18
  198. package/src/config/default-profile-catalog.ts +12 -11
  199. package/src/config/feature-flag-registry.json +22 -13
  200. package/src/config/schema.ts +2 -0
  201. package/src/config/schemas/__tests__/live-voice.test.ts +13 -0
  202. package/src/config/schemas/live-voice.ts +7 -0
  203. package/src/config/schemas/sight.ts +45 -0
  204. package/src/config/skills.ts +2 -1
  205. package/src/contacts/__tests__/mirror-reconciler.test.ts +352 -0
  206. package/src/contacts/contact-store.ts +47 -4
  207. package/src/contacts/contacts-write.ts +5 -2
  208. package/src/contacts/mirror-reconciler.ts +210 -0
  209. package/src/contacts/mirror-tombstones.ts +42 -0
  210. package/src/contacts/notify-contacts-changed.ts +11 -2
  211. package/src/context/compactor.ts +47 -19
  212. package/src/context/tool-result-spool.ts +21 -14
  213. package/src/conversations/message-consolidation.ts +2 -2
  214. package/src/credential-health/credential-health-service.ts +37 -6
  215. package/src/daemon/conversation-agent-loop-handlers.ts +19 -2
  216. package/src/daemon/conversation-agent-loop.ts +100 -1
  217. package/src/daemon/conversation-media-retry.ts +11 -6
  218. package/src/daemon/conversation-messaging.ts +293 -30
  219. package/src/daemon/conversation-runtime-assembly.ts +67 -0
  220. package/src/daemon/conversation-sight-frames.ts +187 -0
  221. package/src/daemon/conversation-store.ts +18 -28
  222. package/src/daemon/conversation-surfaces.ts +12 -2
  223. package/src/daemon/conversation-tool-setup.ts +39 -10
  224. package/src/daemon/conversation.ts +159 -16
  225. package/src/daemon/daemon-control.ts +6 -17
  226. package/src/daemon/handlers/config-model.test.ts +1 -0
  227. package/src/daemon/handlers/conversations.ts +10 -11
  228. package/src/daemon/install-assistant-command.ts +1 -0
  229. package/src/daemon/lifecycle.ts +10 -13
  230. package/src/daemon/message-types/sync.ts +4 -0
  231. package/src/daemon/process-message.ts +2 -17
  232. package/src/daemon/reaction-history-render.test.ts +162 -0
  233. package/src/daemon/reaction-history-render.ts +115 -0
  234. package/src/daemon/reaction-record.test.ts +162 -0
  235. package/src/daemon/reaction-record.ts +83 -0
  236. package/src/daemon/shutdown-handlers.ts +0 -2
  237. package/src/daemon/summarize-boundary.ts +12 -0
  238. package/src/daemon/video-thumbnail.ts +1 -1
  239. package/src/home/feed-types.ts +4 -0
  240. package/src/home/feed-writer.ts +25 -0
  241. package/src/live-voice/__tests__/live-voice-attach-frame.test.ts +679 -0
  242. package/src/live-voice/__tests__/live-voice-attach-image.test.ts +8 -5
  243. package/src/live-voice/__tests__/live-voice-photo.test.ts +637 -1
  244. package/src/live-voice/__tests__/live-voice-sight-frame-inline.test.ts +787 -0
  245. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +347 -0
  246. package/src/live-voice/__tests__/live-voice-vad.test.ts +109 -0
  247. package/src/live-voice/live-voice-photo.ts +420 -65
  248. package/src/live-voice/live-voice-session.ts +311 -27
  249. package/src/live-voice/protocol.ts +147 -11
  250. package/src/messaging/provider-message-metadata.ts +7 -0
  251. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +113 -1
  252. package/src/messaging/providers/channel-transport.ts +37 -0
  253. package/src/messaging/providers/discord/api.ts +4 -4
  254. package/src/messaging/providers/discord/send.test.ts +177 -2
  255. package/src/messaging/providers/discord/send.ts +197 -10
  256. package/src/messaging/providers/discord/transport.ts +10 -0
  257. package/src/messaging/providers/index.ts +42 -0
  258. package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +129 -3
  259. package/src/messaging/providers/slack/__tests__/auth.test.ts +35 -0
  260. package/src/messaging/providers/slack/__tests__/message-content.test.ts +152 -0
  261. package/src/messaging/providers/slack/adapter.ts +112 -27
  262. package/src/messaging/providers/slack/api.test.ts +10 -0
  263. package/src/messaging/providers/slack/auth.ts +32 -6
  264. package/src/messaging/providers/slack/client.ts +0 -15
  265. package/src/messaging/providers/slack/message-content.ts +137 -0
  266. package/src/messaging/providers/slack/send.test.ts +43 -2
  267. package/src/messaging/providers/slack/send.ts +18 -5
  268. package/src/messaging/providers/slack/transport.ts +10 -0
  269. package/src/messaging/providers/slack/types.ts +21 -2
  270. package/src/messaging/providers/telegram-bot/adapter.ts +5 -1
  271. package/src/messaging/providers/telegram-bot/send.test.ts +49 -2
  272. package/src/messaging/providers/telegram-bot/send.ts +41 -1
  273. package/src/messaging/providers/telegram-bot/transport.ts +10 -0
  274. package/src/messaging/providers/telegram-bot/withdraw.ts +13 -5
  275. package/src/messaging/reaction-envelopes.ts +79 -0
  276. package/src/monitoring/control.ts +2 -2
  277. package/src/monitoring/page-cache.ts +5 -1
  278. package/src/notifications/AGENTS.md +1 -1
  279. package/src/notifications/__tests__/guardian-feed-projection.test.ts +360 -0
  280. package/src/notifications/__tests__/home-feed-side-effect.test.ts +8 -0
  281. package/src/notifications/adapters/discord.ts +62 -9
  282. package/src/notifications/conversation-pairing.ts +31 -4
  283. package/src/notifications/emit-signal.ts +8 -5
  284. package/src/notifications/guardian-delivery-recorder.ts +47 -6
  285. package/src/notifications/guardian-feed-projection.ts +492 -0
  286. package/src/notifications/home-feed-side-effect.ts +59 -11
  287. package/src/oauth/byo-connection.test.ts +22 -0
  288. package/src/oauth/byo-connection.ts +11 -5
  289. package/src/oauth/connection.ts +3 -2
  290. package/src/oauth/platform-connection.test.ts +35 -0
  291. package/src/oauth/platform-connection.ts +18 -13
  292. package/src/oauth/scope-utils.ts +48 -0
  293. package/src/oauth/seed-providers.ts +12 -0
  294. package/src/outbound-proxy/certs.ts +5 -4
  295. package/src/permissions/question-guardian-request.test.ts +7 -1
  296. package/src/persistence/attachments-store.ts +134 -31
  297. package/src/persistence/conversation-crud.ts +137 -4
  298. package/src/persistence/conversation-types.ts +109 -0
  299. package/src/persistence/db-connection.ts +3 -21
  300. package/src/persistence/delivery-crud.ts +42 -12
  301. package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +15 -2
  302. package/src/persistence/embeddings/embedding-local.ts +7 -112
  303. package/src/persistence/embeddings/qdrant-manager.ts +4 -1
  304. package/src/persistence/llm-request-log-store.ts +1 -1
  305. package/src/persistence/migrations/370-acp-session-history-auth-error-code.ts +36 -0
  306. package/src/persistence/migrations/371-acp-session-history-auth-error-credential.ts +40 -0
  307. package/src/persistence/migrations/372-create-acp-refused-credentials.ts +34 -0
  308. package/src/persistence/migrations/373-acp-auth-marker-index.ts +27 -0
  309. package/src/persistence/migrations/__tests__/370-acp-session-history-auth-error-code.test.ts +65 -0
  310. package/src/persistence/migrations/__tests__/371-acp-session-history-auth-error-credential.test.ts +68 -0
  311. package/src/persistence/migrations/__tests__/372-create-acp-refused-credentials.test.ts +64 -0
  312. package/src/persistence/migrations/__tests__/373-acp-auth-marker-index.test.ts +82 -0
  313. package/src/persistence/schema/acp.ts +25 -0
  314. package/src/persistence/steps.ts +8 -0
  315. package/src/platform/platform-patch-queue.test.ts +43 -41
  316. package/src/plugin-api/index.ts +12 -0
  317. package/src/plugins/defaults/compaction/window-manager.ts +2 -0
  318. package/src/plugins/defaults/image-recovery/recover.ts +12 -1
  319. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +166 -1
  320. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +3 -0
  321. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +3 -0
  322. package/src/plugins/defaults/memory/jobs-worker.ts +1 -2
  323. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +53 -9
  324. package/src/plugins/defaults/memory/src/__tests__/memory-worker-routes.test.ts +4 -7
  325. package/src/plugins/defaults/memory/src/memory-worker-routes.ts +1 -1
  326. package/src/plugins/defaults/memory/v1/job-handlers/index-maintenance.ts +15 -1
  327. package/src/plugins/defaults/memory/worker-control.ts +1 -2
  328. package/src/providers/content-block-schema.ts +1 -0
  329. package/src/providers/fireworks/client.ts +9 -1
  330. package/src/providers/gemini/client.ts +22 -15
  331. package/src/providers/media-resolve.ts +24 -10
  332. package/src/providers/model-catalog.ts +91 -0
  333. package/src/providers/openai/chat-completions-provider.ts +44 -5
  334. package/src/providers/retry.ts +16 -11
  335. package/src/providers/types.ts +56 -0
  336. package/src/routes/__tests__/control.test.ts +4 -3
  337. package/src/routes/control.ts +1 -1
  338. package/src/routes/route-host-client.ts +1 -1
  339. package/src/runtime/AGENTS.md +30 -12
  340. package/src/runtime/__tests__/agent-wake.test.ts +71 -6
  341. package/src/runtime/__tests__/guardian-form-registry.test.ts +242 -0
  342. package/src/runtime/access-request-helper.ts +15 -7
  343. package/src/runtime/agent-wake.ts +16 -4
  344. package/src/runtime/channel-approval-types.ts +8 -5
  345. package/src/runtime/channel-invite-transport.ts +2 -0
  346. package/src/runtime/channel-invite-transports/discord.ts +27 -0
  347. package/src/runtime/channel-readiness-service.ts +4 -4
  348. package/src/runtime/channel-reply-delivery.ts +72 -28
  349. package/src/runtime/guardian-action-service.ts +9 -5
  350. package/src/runtime/guardian-form-registry.ts +256 -0
  351. package/src/runtime/guardian-reply-router.ts +33 -14
  352. package/src/runtime/invite-instruction-generator.ts +14 -2
  353. package/src/runtime/migrations/vbundle-metadata-merge.ts +4 -7
  354. package/src/runtime/no-response.ts +13 -74
  355. package/src/runtime/routes/__tests__/acp-routes.test.ts +527 -0
  356. package/src/runtime/routes/__tests__/app-management-shared-apps.test.ts +74 -0
  357. package/src/runtime/routes/__tests__/contact-form-kinds.test.ts +22 -0
  358. package/src/runtime/routes/__tests__/contact-record-prompt-route.test.ts +303 -0
  359. package/src/runtime/routes/__tests__/monitoring-routes.test.ts +3 -4
  360. package/src/runtime/routes/__tests__/pending-interactions-route.test.ts +6 -0
  361. package/src/runtime/routes/__tests__/route-host-worker-routes.test.ts +3 -4
  362. package/src/runtime/routes/__tests__/schedule-worker-routes.test.ts +5 -6
  363. package/src/runtime/routes/acp-routes-list.test.ts +58 -0
  364. package/src/runtime/routes/acp-routes.ts +253 -47
  365. package/src/runtime/routes/app-management-routes.ts +80 -47
  366. package/src/runtime/routes/approval-routes.ts +9 -0
  367. package/src/runtime/routes/archive-utils.ts +6 -1
  368. package/src/runtime/routes/avatar-routes.ts +1 -1
  369. package/src/runtime/routes/channel-route-shared.ts +18 -20
  370. package/src/runtime/routes/contact-prompt-routes.ts +271 -105
  371. package/src/runtime/routes/conversation-routes.ts +41 -32
  372. package/src/runtime/routes/credential-prompt-routes.ts +10 -1
  373. package/src/runtime/routes/debug-bash-routes.ts +1 -0
  374. package/src/runtime/routes/guardian-expiry-sweep.ts +89 -31
  375. package/src/runtime/routes/guardian-form-routes.ts +115 -0
  376. package/src/runtime/routes/inbound-message-handler.ts +82 -29
  377. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -3
  378. package/src/runtime/routes/inbound-stages/edit-intercept.ts +6 -0
  379. package/src/runtime/routes/inbound-stages/guardian-reply-intercept.test.ts +6 -0
  380. package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +6 -1
  381. package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +50 -17
  382. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +97 -94
  383. package/src/runtime/routes/index.ts +2 -0
  384. package/src/runtime/routes/integrations/slack/channel.ts +39 -18
  385. package/src/runtime/routes/migration-routes.ts +1 -3
  386. package/src/runtime/routes/monitoring-routes.ts +0 -3
  387. package/src/runtime/routes/oauth-commands-routes.ts +18 -5
  388. package/src/runtime/routes/route-host-worker-routes.ts +1 -1
  389. package/src/runtime/routes/schedule-worker-routes.ts +1 -1
  390. package/src/runtime/tool-grant-request-helper.ts +4 -0
  391. package/src/schedule/run-script.ts +1 -0
  392. package/src/schedule/scheduler.ts +1 -1
  393. package/src/schedule/worker-control.ts +2 -2
  394. package/src/security/oauth2.ts +2 -11
  395. package/src/security/secure-keys.ts +55 -8
  396. package/src/skills/catalog-install.ts +7 -4
  397. package/src/skills/clawhub.ts +1 -0
  398. package/src/skills/inline-command-runner.ts +1 -0
  399. package/src/stt/__tests__/room-noise-floor.test.ts +108 -0
  400. package/src/stt/room-noise-floor.ts +140 -0
  401. package/src/subagent/notify.ts +49 -3
  402. package/src/tools/__tests__/tool-input-schemas.test.ts +2 -0
  403. package/src/tools/acp/spawn.ts +2 -2
  404. package/src/tools/browser/__tests__/browser-status.test.ts +9 -0
  405. package/src/tools/browser/browser-execution.ts +10 -43
  406. package/src/tools/browser/browser-manager.ts +1 -0
  407. package/src/tools/browser/browser-status-constants.ts +6 -5
  408. package/src/tools/browser/runtime-check.ts +2 -0
  409. package/src/tools/channel/react-to-message.test.ts +193 -0
  410. package/src/tools/channel/react-to-message.ts +159 -0
  411. package/src/tools/credentials/metadata-store.ts +115 -19
  412. package/src/tools/credentials/store.ts +0 -2
  413. package/src/tools/filesystem/read.ts +2 -2
  414. package/src/tools/host-filesystem/read.ts +2 -2
  415. package/src/tools/host-terminal/host-shell.test.ts +24 -0
  416. package/src/tools/host-terminal/host-shell.ts +11 -38
  417. package/src/tools/shared/filesystem/file-ops-service.ts +10 -7
  418. package/src/tools/shared/shell-output.test.ts +104 -0
  419. package/src/tools/shared/shell-output.ts +112 -39
  420. package/src/tools/skills/sandbox-runner.ts +1 -0
  421. package/src/tools/system/request-permission.test.ts +34 -0
  422. package/src/tools/system/request-permission.ts +29 -5
  423. package/src/tools/terminal/shell.test.ts +63 -0
  424. package/src/tools/terminal/shell.ts +12 -44
  425. package/src/tools/tool-input-schemas.ts +2 -0
  426. package/src/tools/tool-manifest.ts +2 -0
  427. package/src/usage/attribution.ts +12 -4
  428. package/src/util/__tests__/worker-ownership.test.ts +77 -0
  429. package/src/util/__tests__/worker-process-command.test.ts +246 -1
  430. package/src/util/bun-runtime.ts +44 -13
  431. package/src/util/bundled-asset.ts +9 -1
  432. package/src/util/disk-usage.ts +1 -0
  433. package/src/util/file-use.ts +4 -1
  434. package/src/util/guardian-form-timeouts.ts +50 -0
  435. package/src/util/image-conversion.ts +1 -0
  436. package/src/util/oauth-request-body.test.ts +59 -0
  437. package/src/util/oauth-request-body.ts +56 -5
  438. package/src/util/platform.ts +69 -73
  439. package/src/util/process-table.ts +16 -5
  440. package/src/util/read-stdin.ts +5 -0
  441. package/src/util/spawn.ts +1 -0
  442. package/src/util/sqlite3-runtime.ts +25 -5
  443. package/src/util/worker-ownership.ts +193 -0
  444. package/src/util/worker-process.ts +375 -37
  445. package/src/workspace/git-service.ts +4 -12
  446. package/src/workspace/migrations/010-app-dir-rename.ts +1 -0
@@ -45,6 +45,10 @@ import {
45
45
  recordLiveVoiceSessionStarted,
46
46
  } from "../onboarding/onboarding-events-store.js";
47
47
  import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
48
+ import {
49
+ attachmentExists,
50
+ deleteOrphanAttachments,
51
+ } from "../persistence/attachments-store.js";
48
52
  import { ensureConversationExists } from "../persistence/conversation-crud.js";
49
53
  import {
50
54
  listProviderIds,
@@ -60,6 +64,7 @@ import {
60
64
  voteDominantLanguage,
61
65
  } from "../stt/language-metadata.js";
62
66
  import { sttCatalogKeyForRole } from "../stt/roles.js";
67
+ import { RoomNoiseFloor } from "../stt/room-noise-floor.js";
63
68
  import {
64
69
  DEFAULT_SPEECH_ENERGY_THRESHOLD,
65
70
  pcm16MaxNormalizedCorrelation,
@@ -106,7 +111,10 @@ import {
106
111
  type VoiceEndpointAction,
107
112
  type VoiceEndpointSource,
108
113
  } from "./live-voice-metrics.js";
109
- import { persistLiveVoicePhoto } from "./live-voice-photo.js";
114
+ import {
115
+ persistLiveVoicePhoto,
116
+ persistLiveVoiceSightFrame,
117
+ } from "./live-voice-photo.js";
110
118
  import {
111
119
  type LiveVoiceSession as LiveVoiceSessionContract,
112
120
  type LiveVoiceSessionCloseReason,
@@ -130,8 +138,10 @@ import {
130
138
  PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
131
139
  } from "./progress-phrases.js";
132
140
  import {
141
+ type LiveVoiceClientAttachFrameFrame,
133
142
  type LiveVoiceClientAttachImageFrame,
134
143
  type LiveVoiceClientFrame,
144
+ type LiveVoiceClientSightFrameFrame,
135
145
  type LiveVoiceClientTextTurnFrame,
136
146
  type LiveVoiceClientUpdateConfigFrame,
137
147
  LiveVoiceProtocolErrorCode,
@@ -200,6 +210,18 @@ const ECHO_ONSET_ELIGIBILITY_MS = 300;
200
210
  // Client buffering makes audible playback trail the server's send-time
201
211
  // estimate. Keep the echo window open briefly past that estimate.
202
212
  const DEFAULT_ECHO_DRAIN_SLACK_MS = 300;
213
+ // The speech gate's base threshold is raised to this multiple of the room's
214
+ // measured noise floor. Speech has to stand clear of the room it is spoken in,
215
+ // and roughly 3x (about 10dB) is the separation an ordinary talking voice keeps
216
+ // over its own background. Mirrors the liveVoice.vad.noiseFloorMargin default.
217
+ const DEFAULT_NOISE_FLOOR_MARGIN = 3;
218
+ // Ceiling on the adaptive base, as a multiple of the configured threshold. The
219
+ // floor estimate is derived from the microphone, so a pathological room (or a
220
+ // stuck input) could in principle drive it high enough that nothing counts as
221
+ // speech and the user cannot interrupt at all. Barge-in getting harder is a
222
+ // nuisance; barge-in becoming impossible is a broken product, so the adaptation
223
+ // is bounded even though the bound should never be reached.
224
+ const NOISE_FLOOR_MAX_BASE_MULTIPLE = 4;
203
225
  // Mirrors MediaTurnDetector's DEFAULT_SILENCE_THRESHOLD_MS: the session
204
226
  // tracks the effective trailing-silence threshold (the detector keeps its own
205
227
  // copy private) so the endpoint decider can report the pause length.
@@ -329,6 +351,14 @@ export interface LiveVoiceSessionOptions {
329
351
  * `DEFAULT_SPEECH_ENERGY_THRESHOLD`.
330
352
  */
331
353
  speechEnergyThreshold?: number;
354
+ /**
355
+ * Multiple of the room's measured noise floor that the speech gate's base
356
+ * threshold is raised to. The production factory seeds this from
357
+ * `liveVoice.vad.noiseFloorMargin` config when unset; defaults to
358
+ * `DEFAULT_NOISE_FLOOR_MARGIN`. Values at or below 0 disable the adaptation,
359
+ * pinning the gate to `speechEnergyThreshold` (used by fixed-gate tests).
360
+ */
361
+ noiseFloorMargin?: number;
332
362
  /**
333
363
  * Sustained speech (ms) required before speech during assistant playback
334
364
  * interrupts it (barge-in); 0 disables the guard. The production factory
@@ -716,6 +746,14 @@ interface ActiveAssistantTurn {
716
746
  // that request's transcript, so the model can tell the user the work is
717
747
  // still running instead of appearing to have dropped it.
718
748
  handedOffRequest: string | null;
749
+ // The parked camera frame this turn claimed, hung on the turn's own
750
+ // persisted user message. Held here so a rolled-back dispatch can hand it
751
+ // back to the session (see restorePendingTurnContext).
752
+ turnAttachmentId: string | null;
753
+ // The frame the rollback could NOT hand back, because a newer one took the
754
+ // slot while this turn was in flight. Nothing will ever send it, so it is
755
+ // collected once the discard has removed the row still linking it.
756
+ staleFrameId: string | null;
719
757
  // The queued announcement this turn consumed alongside `continuationResult`
720
758
  // — the two are routes for one answer, so launching consumes both. Held here
721
759
  // so a rolled-back speculative dispatch can re-arm it (see
@@ -1176,6 +1214,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1176
1214
  // Base energy gate for server-VAD speech classification. During estimated
1177
1215
  // playback, classifyVadEnergy raises this above the learned echo level.
1178
1216
  private readonly speechEnergyThreshold: number | undefined;
1217
+ private readonly noiseFloorMargin: number;
1218
+ // Quietest recent second of microphone audio, learned only while the
1219
+ // assistant is silent (during playback the microphone hears echo, which is
1220
+ // what echoEnergyEma is for). Raises the base gate in a noisy room so the
1221
+ // room itself stops reading as speech.
1222
+ private readonly roomNoiseFloor = new RoomNoiseFloor();
1179
1223
  private readonly echoBargeInMargin: number;
1180
1224
  private readonly echoEmaHalfLifeMs: number;
1181
1225
  private readonly echoDrainSlackMs: number;
@@ -1255,6 +1299,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1255
1299
  // turn; cleared when the continuation finishes (by then the result note
1256
1300
  // takes over and "still running" would be stale).
1257
1301
  private pendingHandoffRequest: string | null = null;
1302
+ // The camera frame a client parked with `attach_frame`, waiting for the next
1303
+ // turn to ride. One slot, latest wins: the frame is ambient context, so an
1304
+ // older one is simply out of date. Consumed (and cleared) when a turn
1305
+ // launches, handed back if that turn is rolled back, and cleared on close.
1306
+ private pendingTurnAttachmentId: string | null = null;
1258
1307
  private readonly maxPendingAudioBytes: number;
1259
1308
  // Set on VAD speech onset; consumed when the first speech chunk is routed
1260
1309
  // to an utterance so the metric lands on the right turn.
@@ -1388,6 +1437,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1388
1437
  ...(options.metricsClock ? { clock: options.metricsClock } : {}),
1389
1438
  });
1390
1439
  this.speechEnergyThreshold = options.speechEnergyThreshold;
1440
+ this.noiseFloorMargin =
1441
+ options.noiseFloorMargin ?? DEFAULT_NOISE_FLOOR_MARGIN;
1391
1442
  // Precedence for the two sensitivity knobs: per-session start-frame
1392
1443
  // override (the client's user setting) > daemon `liveVoice.vad` config
1393
1444
  // (seeded into `options` by the factory) > in-code default.
@@ -1526,6 +1577,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1526
1577
  case "attach_image":
1527
1578
  this.persistPhoto(frame);
1528
1579
  return;
1580
+ case "attach_frame":
1581
+ this.parkTurnFrame(frame);
1582
+ return;
1583
+ case "sight_frame":
1584
+ this.persistSightFrame(frame);
1585
+ return;
1529
1586
  case "text":
1530
1587
  await this.handleTextTurn(frame);
1531
1588
  return;
@@ -1618,6 +1675,138 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1618
1675
  );
1619
1676
  }
1620
1677
 
1678
+ /**
1679
+ * Persist a kept camera frame into the conversation, running no turn.
1680
+ *
1681
+ * Fire-and-forget for the same reason the photo is: the persist waits out any
1682
+ * in-flight turn, and the socket must keep pumping audio meanwhile. Waiting
1683
+ * is also what keeps a frame that lands mid-reply from splitting that reply's
1684
+ * own rows.
1685
+ *
1686
+ * The frame becomes its own tagged user message rather than riding a turn, so
1687
+ * the transcript is the record of what the assistant saw and the model
1688
+ * correlates a frame with speech by adjacency. See `live-voice-photo.ts`.
1689
+ *
1690
+ * An id the attachment store cannot resolve is refused there, before the
1691
+ * persist waits on anything, so a bad frame is answered while the client can
1692
+ * still act on it rather than a turn's length later. That resolve is the only
1693
+ * existence check: a second one here would read the same row to learn the
1694
+ * same fact.
1695
+ */
1696
+ private persistSightFrame(frame: LiveVoiceClientSightFrameFrame): void {
1697
+ void persistLiveVoiceSightFrame(
1698
+ this.conversationId,
1699
+ frame.attachmentId,
1700
+ ).then((result) => {
1701
+ if (!result.ok && !this.isClosed) {
1702
+ void this.sendFrame({
1703
+ type: "error",
1704
+ code: LiveVoiceProtocolErrorCode.InvalidFrame,
1705
+ message: "Could not attach that camera frame to the conversation.",
1706
+ // Names the frame as the casualty so the client can retract the
1707
+ // preview it already showed, rather than filing this with the
1708
+ // transient transcriber and TTS blips that share `recoverable`.
1709
+ frameType: "sight_frame",
1710
+ // The session is fine; only this frame failed.
1711
+ recoverable: true,
1712
+ });
1713
+ }
1714
+ });
1715
+ }
1716
+
1717
+ /**
1718
+ * Park a camera frame for the next turn to carry, or give up the parked one.
1719
+ *
1720
+ * Nothing is persisted and no turn runs: the frame is ambient context, and
1721
+ * on its own it says nothing the user asked to say. It rides the next turn's
1722
+ * own user message instead, so the picture and the words about it are one
1723
+ * message. A frame that arrives while a turn is already running belongs to
1724
+ * the turn after it, which the drain at launch takes care of.
1725
+ *
1726
+ * A null id unparks: the client's camera is off, so the slot must stop
1727
+ * answering for a view nobody can see any more. The frame goes with it, and
1728
+ * unparking an empty slot is a no-op.
1729
+ *
1730
+ * The id is checked against the attachment store here rather than at the
1731
+ * turn, for the same reason a failed photo is reported: an id that resolves
1732
+ * to nothing would otherwise leave the user believing the assistant can see
1733
+ * something it never received, one turn later and with nothing said.
1734
+ */
1735
+ private parkTurnFrame(frame: LiveVoiceClientAttachFrameFrame): void {
1736
+ if (frame.attachmentId === null) {
1737
+ const parked = this.pendingTurnAttachmentId;
1738
+ this.pendingTurnAttachmentId = null;
1739
+ this.reclaimParkedFrame(parked);
1740
+ return;
1741
+ }
1742
+
1743
+ let exists = false;
1744
+ try {
1745
+ exists = attachmentExists(frame.attachmentId);
1746
+ } catch (err) {
1747
+ log.warn(
1748
+ { err, attachmentId: frame.attachmentId },
1749
+ "Live-voice camera frame lookup failed",
1750
+ );
1751
+ }
1752
+ if (!exists) {
1753
+ void this.sendFrame({
1754
+ type: "error",
1755
+ code: LiveVoiceProtocolErrorCode.InvalidFrame,
1756
+ message: "Could not attach that camera frame to the conversation.",
1757
+ frameType: "attach_frame",
1758
+ // The session is fine; only this frame failed.
1759
+ recoverable: true,
1760
+ });
1761
+ return;
1762
+ }
1763
+ const displaced = this.pendingTurnAttachmentId;
1764
+ this.pendingTurnAttachmentId = frame.attachmentId;
1765
+ if (displaced !== null && displaced !== frame.attachmentId) {
1766
+ this.reclaimParkedFrame(displaced);
1767
+ }
1768
+ }
1769
+
1770
+ /**
1771
+ * Give up a parked camera frame no turn will ever carry: one a newer frame
1772
+ * displaced, one the session end leaves behind, one a rollback could not
1773
+ * hand back. A client shooting the camera on a timer parks one every few
1774
+ * seconds, and attachment collection is scoped to message deletion with no
1775
+ * global sweep, so an id dropped without this leaves its row and its bytes
1776
+ * behind for good.
1777
+ *
1778
+ * Safe because `deleteOrphanAttachments` is link-aware: it removes only ids
1779
+ * with no message link, so a frame that already rode a turn is kept and one
1780
+ * that never left this slot is collected. Nothing waits on the result and a
1781
+ * store that cannot be reached must not take the call down, so a failure is
1782
+ * logged and dropped.
1783
+ */
1784
+ private reclaimParkedFrame(attachmentId: string | null): void {
1785
+ if (attachmentId === null) {
1786
+ return;
1787
+ }
1788
+ try {
1789
+ deleteOrphanAttachments([attachmentId]);
1790
+ } catch (err) {
1791
+ log.warn(
1792
+ { err, attachmentId },
1793
+ "Could not reclaim a parked live-voice camera frame",
1794
+ );
1795
+ }
1796
+ }
1797
+
1798
+ /**
1799
+ * Collect the frame a turn's rollback could not hand back, once the discard
1800
+ * that owns the turn's row has finished. Ordering is the whole point: the
1801
+ * reclaim is link-aware, so running it while the doomed row still links the
1802
+ * frame finds the link, keeps the frame, and leaks it.
1803
+ */
1804
+ private reclaimStaleTurnFrame(turn: ActiveAssistantTurn): void {
1805
+ const staleFrameId = turn.staleFrameId;
1806
+ turn.staleFrameId = null;
1807
+ this.reclaimParkedFrame(staleFrameId);
1808
+ }
1809
+
1621
1810
  /**
1622
1811
  * Apply a mid-session `update_config` frame: retune the live turn detector's
1623
1812
  * pause ("pause before reply") and/or the barge-in guard ("interrupt
@@ -1681,6 +1870,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1681
1870
  this.turnDetector?.dispose();
1682
1871
  this.clearEndpointExtensionTimer();
1683
1872
  this.clearProviderTurnEndTimer();
1873
+ // No turn will ever carry it now, and the frame is the client's to shoot
1874
+ // again on its next session, so the staged row and bytes go with the call.
1875
+ this.reclaimParkedFrame(this.pendingTurnAttachmentId);
1876
+ this.pendingTurnAttachmentId = null;
1684
1877
  // There is no longer anyone on the call to speak to, so a queued
1685
1878
  // announcement cannot be spoken — but the answer behind it is finished
1686
1879
  // work the user asked for, so it takes the same conversation route a
@@ -2166,18 +2359,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2166
2359
  * confirmed echo while speech above the learned margin remains frozen out.
2167
2360
  */
2168
2361
  private classifyVadEnergy(chunk: Buffer): VadClassifiedChunk[] {
2169
- const baseThreshold =
2170
- this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
2171
2362
  const meanAmplitude = pcm16MeanAmplitude(chunk);
2363
+ const chunkMs = pcm16DurationMs(
2364
+ chunk.byteLength,
2365
+ this.context.startFrame.audio.sampleRate,
2366
+ );
2367
+ const baseThreshold = this.effectiveBaseThreshold();
2172
2368
  if (
2173
2369
  this.echoBargeInMargin <= 1 ||
2174
2370
  !this.isAssistantPlaybackEchoPossible()
2175
2371
  ) {
2176
2372
  this.resetEchoReference();
2373
+ // Only learn the room while the assistant is silent. The floor is a
2374
+ // minimum over completed blocks, so this chunk cannot lower the
2375
+ // threshold that is about to judge it.
2376
+ this.roomNoiseFloor.observe(meanAmplitude, chunkMs);
2177
2377
  return [
2178
2378
  this.classifyAtFixedThreshold(chunk, baseThreshold, meanAmplitude),
2179
2379
  ];
2180
2380
  }
2381
+ // Playback is starting or ongoing: stop measuring the room, and drop the
2382
+ // partial block rather than let it straddle the silence and the echo.
2383
+ this.roomNoiseFloor.interrupt();
2181
2384
 
2182
2385
  if (this.echoWindowTotalAudioMs === 0) {
2183
2386
  this.echoWindowGuardCarryover =
@@ -2186,10 +2389,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2186
2389
  this.echoWindowGuardCarryover = false;
2187
2390
  }
2188
2391
 
2189
- const chunkMs = pcm16DurationMs(
2190
- chunk.byteLength,
2191
- this.context.startFrame.audio.sampleRate,
2192
- );
2193
2392
  const onsetWasEligible =
2194
2393
  !this.echoOnsetLapsed &&
2195
2394
  this.echoWindowTotalAudioMs < ECHO_ONSET_ELIGIBILITY_MS;
@@ -2296,6 +2495,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2296
2495
  alpha * meanAmplitude + (1 - alpha) * this.echoEnergyEma;
2297
2496
  }
2298
2497
 
2498
+ /**
2499
+ * The gate a chunk is judged against before any playback-echo adjustment.
2500
+ *
2501
+ * The configured `speechEnergyThreshold` is a floor, never a ceiling: the
2502
+ * adaptation can only make speech classification *stricter*. That asymmetry
2503
+ * is deliberate. A gate that could drop below the configured value would be
2504
+ * able to invent new self-interruption in a room quieter than the one the
2505
+ * constant was chosen for, which is the failure this whole change exists to
2506
+ * remove; a gate that only rises can, at worst, make a soft barge-in need
2507
+ * repeating. The result is bounded so it can never rise far enough to make
2508
+ * interrupting impossible.
2509
+ */
2510
+ private effectiveBaseThreshold(): number {
2511
+ const configured =
2512
+ this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
2513
+ if (this.noiseFloorMargin <= 0) {
2514
+ return configured;
2515
+ }
2516
+ const floor = this.roomNoiseFloor.floor;
2517
+ if (floor === null) {
2518
+ return configured;
2519
+ }
2520
+ return Math.min(
2521
+ Math.max(configured, floor * this.noiseFloorMargin),
2522
+ configured * NOISE_FLOOR_MAX_BASE_MULTIPLE,
2523
+ );
2524
+ }
2525
+
2299
2526
  private classifyAtFixedThreshold(
2300
2527
  chunk: Buffer,
2301
2528
  baseThreshold: number,
@@ -3852,9 +4079,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3852
4079
  ): void {
3853
4080
  turn.speculativePending = false;
3854
4081
  // The dispatch is being unwound, so the barge-in merge note, the finished
3855
- // continuation's answer, its queued announcement, and any photos it claimed
3856
- // all go back to the session. The turn that would have delivered them is
3857
- // gone, and the utterance they belong to is about to be sent by another.
4082
+ // continuation's answer, its queued announcement, and any camera frame it
4083
+ // claimed all go back to the session. The turn that would have delivered
4084
+ // them is gone, and the utterance they belong to is about to be sent by
4085
+ // another.
3858
4086
  this.restorePendingTurnContext(turn);
3859
4087
  // Latched before the handle check: when the discard beats the bridge
3860
4088
  // handle's resolution (startVoiceTurn still persisting), the handle's
@@ -3869,12 +4097,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3869
4097
  );
3870
4098
  const handle = turn.handle;
3871
4099
  turn.handle = null;
3872
- void handle?.discard?.().catch((err: unknown) => {
3873
- log.warn(
3874
- { err, turnId: turn.turnId, reason },
3875
- "Speculative voice turn discard failed",
3876
- );
3877
- });
4100
+ void handle
4101
+ ?.discard?.()
4102
+ .catch((err: unknown) => {
4103
+ log.warn(
4104
+ { err, turnId: turn.turnId, reason },
4105
+ "Speculative voice turn discard failed",
4106
+ );
4107
+ })
4108
+ // Only once the discard has deleted the row: until then the row still
4109
+ // links a displaced frame, and the reclaim would keep it forever.
4110
+ .finally(() => {
4111
+ this.reclaimStaleTurnFrame(turn);
4112
+ });
3878
4113
  turn.utterance.assistantTurnStarted = false;
3879
4114
  log.info(
3880
4115
  { turnId: turn.turnId, reason },
@@ -4622,10 +4857,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4622
4857
  }
4623
4858
 
4624
4859
  /**
4625
- * Take the barge-in merge context, the completed-continuation context, and
4626
- * the handoff note for the turn being launched — each feeds exactly the next
4627
- * launched turn — and cancel the queued announcement, because the turn this
4628
- * context rides delivers the same answer and would otherwise say it twice.
4860
+ * Take the barge-in merge context, the completed-continuation context, the
4861
+ * handoff note, and the parked camera frame for the turn being launched
4862
+ * (each feeds exactly the next launched turn) and cancel the queued
4863
+ * announcement, because the turn this context rides delivers the same answer
4864
+ * and would otherwise say it twice.
4629
4865
  *
4630
4866
  * Every real user turn consumes here, speculative or not: a speculative
4631
4867
  * dispatch latches `assistantTurnStarted`, so a turn that skipped this would
@@ -4637,16 +4873,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4637
4873
  continuationResult: string | null;
4638
4874
  handedOffRequest: string | null;
4639
4875
  announcement: ContinuationDelivery | null;
4876
+ turnAttachmentId: string | null;
4640
4877
  } {
4641
4878
  const consumed = {
4642
4879
  interruptedRequest: this.pendingInterruptedRequest,
4643
4880
  continuationResult: this.pendingContinuationResult,
4644
4881
  handedOffRequest: this.pendingHandoffRequest,
4645
4882
  announcement: this.pendingAnnouncement,
4883
+ turnAttachmentId: this.pendingTurnAttachmentId,
4646
4884
  };
4647
4885
  this.pendingInterruptedRequest = null;
4648
4886
  this.pendingContinuationResult = null;
4649
4887
  this.pendingHandoffRequest = null;
4888
+ this.pendingTurnAttachmentId = null;
4650
4889
  this.clearContinuationAnnouncement();
4651
4890
  return consumed;
4652
4891
  }
@@ -4668,21 +4907,43 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4668
4907
  const continuationResult = turn.continuationResult;
4669
4908
  const handedOffRequest = turn.handedOffRequest;
4670
4909
  const announcement = turn.consumedAnnouncement;
4910
+ const turnAttachmentId = turn.turnAttachmentId;
4671
4911
  // One restore per turn: the rollback paths overlap (a discarded turn whose
4672
4912
  // leg start also fails), and the second pass must be a no-op.
4673
4913
  turn.interruptedRequest = null;
4674
4914
  turn.continuationResult = null;
4675
4915
  turn.handedOffRequest = null;
4676
4916
  turn.consumedAnnouncement = null;
4917
+ turn.turnAttachmentId = null;
4918
+ // Nobody's to send unless the slot already holds it. A client that
4919
+ // retransmits the id this turn claimed parked the SAME frame rather than a
4920
+ // newer one, and collecting it would leave the slot pointing at a row that
4921
+ // is gone: the discard keeps the bytes alive precisely so the replay can
4922
+ // still send them.
4923
+ const orphanedFrameId =
4924
+ turnAttachmentId !== null &&
4925
+ this.pendingTurnAttachmentId !== turnAttachmentId
4926
+ ? turnAttachmentId
4927
+ : null;
4677
4928
  if (
4678
4929
  this.isClosed ||
4679
4930
  this.detachStopGeneration !== turn.pendingContextStopGeneration
4680
4931
  ) {
4932
+ // The drop is deliberate, so the frame it drops is nobody's to send.
4933
+ turn.staleFrameId = orphanedFrameId;
4681
4934
  return;
4682
4935
  }
4683
4936
  if (this.pendingInterruptedRequest === null) {
4684
4937
  this.pendingInterruptedRequest = interruptedRequest;
4685
4938
  }
4939
+ // A frame parked since this turn launched is newer, so it wins outright
4940
+ // and the one being handed back is the stale one: collect it rather than
4941
+ // leave it staged for a turn that will never ask for it.
4942
+ if (this.pendingTurnAttachmentId === null) {
4943
+ this.pendingTurnAttachmentId = turnAttachmentId;
4944
+ } else {
4945
+ turn.staleFrameId = orphanedFrameId;
4946
+ }
4686
4947
  if (this.pendingContinuationResult === null) {
4687
4948
  this.pendingContinuationResult = continuationResult;
4688
4949
  }
@@ -4779,6 +5040,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4779
5040
  interruptedRequest: pending?.interruptedRequest ?? null,
4780
5041
  continuationResult: pending?.continuationResult ?? null,
4781
5042
  handedOffRequest: pending?.handedOffRequest ?? null,
5043
+ turnAttachmentId: pending?.turnAttachmentId ?? null,
5044
+ staleFrameId: null,
4782
5045
  consumedAnnouncement: pending?.announcement ?? null,
4783
5046
  pendingContextStopGeneration: this.detachStopGeneration,
4784
5047
  continuationDelivery: opts?.continuationDelivery ?? null,
@@ -4803,6 +5066,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4803
5066
  await this.sendFrame({ type: "thinking", turnId });
4804
5067
  if (!this.isActiveAssistantTurn(token)) {
4805
5068
  this.restorePendingTurnContext(activeTurn);
5069
+ // Nothing was persisted, so a frame the restore could not hand back
5070
+ // links to no row and is collectible right here.
5071
+ this.reclaimStaleTurnFrame(activeTurn);
4806
5072
  return false;
4807
5073
  }
4808
5074
  } else {
@@ -4853,11 +5119,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4853
5119
  content,
4854
5120
  routingLeg: "front-door",
4855
5121
  frontDoor: true,
5122
+ // Only this leg: an escalated leg persists its own continuation row, and
5123
+ // the frame belongs to the row carrying the user's words.
5124
+ ...(activeTurn.turnAttachmentId
5125
+ ? { attachments: [activeTurn.turnAttachmentId] }
5126
+ : {}),
4856
5127
  });
4857
5128
  if (!started) {
4858
5129
  // Nothing was persisted or spoken, so the context this turn consumed goes
4859
- // back to the session rather than dying with the failed dispatch.
5130
+ // back to the session rather than dying with the failed dispatch. A frame
5131
+ // a newer one displaced meanwhile links to no row, so it is collected
5132
+ // here rather than left staged.
4860
5133
  this.restorePendingTurnContext(activeTurn);
5134
+ this.reclaimStaleTurnFrame(activeTurn);
4861
5135
  }
4862
5136
  return started;
4863
5137
  }
@@ -4890,6 +5164,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4890
5164
  routingLeg?: VoiceRoutingLeg;
4891
5165
  frontDoor?: boolean;
4892
5166
  spokenEscalationBridge?: string;
5167
+ attachments?: readonly string[];
4893
5168
  },
4894
5169
  ): Promise<boolean> {
4895
5170
  if (!this.startVoiceTurn) {
@@ -4994,6 +5269,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4994
5269
  this.clearAwaitingApproval(activeTurn);
4995
5270
  },
4996
5271
  content: leg.content,
5272
+ ...(leg.attachments ? { attachments: leg.attachments } : {}),
4997
5273
  isInbound: true,
4998
5274
  launchedAtMs: activeTurn.launchedAtMs,
4999
5275
  signal: activeTurn.abortController.signal,
@@ -5310,12 +5586,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5310
5586
  // rollback: a plain abort would leave the discarded pause's user
5311
5587
  // row in history.
5312
5588
  if (activeTurn.discardRequested && handle.discard) {
5313
- void handle.discard().catch((err: unknown) => {
5314
- log.warn(
5315
- { err, turnId: activeTurn.turnId },
5316
- "Late speculative voice turn discard failed",
5317
- );
5318
- });
5589
+ void handle
5590
+ .discard()
5591
+ .catch((err: unknown) => {
5592
+ log.warn(
5593
+ { err, turnId: activeTurn.turnId },
5594
+ "Late speculative voice turn discard failed",
5595
+ );
5596
+ })
5597
+ // This discard owns the row, so it owns the reclaim of any frame
5598
+ // the rollback could not hand back (see reclaimStaleTurnFrame).
5599
+ .finally(() => {
5600
+ this.reclaimStaleTurnFrame(activeTurn);
5601
+ });
5319
5602
  } else {
5320
5603
  handle.abort();
5321
5604
  }
@@ -6657,6 +6940,7 @@ export function createLiveVoiceSession(
6657
6940
  : {}),
6658
6941
  speechEnergyThreshold:
6659
6942
  options.speechEnergyThreshold ?? vadConfig?.speechEnergyThreshold,
6943
+ noiseFloorMargin: options.noiseFloorMargin ?? vadConfig?.noiseFloorMargin,
6660
6944
  bargeInMinSpeechMs:
6661
6945
  options.bargeInMinSpeechMs ?? vadConfig?.bargeInMinSpeechMs,
6662
6946
  echoBargeInMargin: