@vellumai/assistant 0.10.12 → 0.11.0-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (809) hide show
  1. package/AGENTS.md +1 -1
  2. package/ARCHITECTURE.md +4 -5
  3. package/README.md +1 -1
  4. package/docs/architecture/memory.md +60 -23
  5. package/docs/error-handling.md +3 -3
  6. package/docs/guardian-request-flow.md +112 -0
  7. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  8. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts +28 -0
  13. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts.map +1 -1
  14. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts +2 -2
  15. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts.map +1 -1
  16. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.js.map +1 -1
  17. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts +2 -0
  18. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts.map +1 -1
  19. package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/package.json +1 -1
  20. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
  25. package/node_modules/@vellumai/gateway-client/package.json +1 -1
  26. package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +6 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +34 -0
  28. package/node_modules/@vellumai/service-contracts/package.json +1 -0
  29. package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
  30. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
  31. package/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
  32. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
  33. package/openapi.yaml +264 -14
  34. package/package.json +2 -2
  35. package/src/__tests__/acp-session.test.ts +3 -3
  36. package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +3 -0
  37. package/src/__tests__/anthropic-provider.test.ts +40 -0
  38. package/src/__tests__/app-conversation-ids-backfill.test.ts +33 -2
  39. package/src/__tests__/app-conversation-ids.test.ts +204 -0
  40. package/src/__tests__/approval-cascade.test.ts +5 -5
  41. package/src/__tests__/approval-routes-http.test.ts +3 -3
  42. package/src/__tests__/ask-question-resolver.test.ts +207 -0
  43. package/src/__tests__/assistant-event-hub-self-exclusion.test.ts +16 -16
  44. package/src/__tests__/assistant-event-hub-targeted.test.ts +13 -11
  45. package/src/__tests__/assistant-event-hub.test.ts +13 -11
  46. package/src/__tests__/assistant-event.test.ts +12 -9
  47. package/src/__tests__/assistant-feature-flag-guard.test.ts +3 -3
  48. package/src/__tests__/assistant-feature-flag-guardrails.test.ts +1 -1
  49. package/src/__tests__/assistant-id-boundary-guard.test.ts +0 -58
  50. package/src/__tests__/assistant-stream-state.test.ts +17 -7
  51. package/src/__tests__/avatar-identity-sync.test.ts +3 -3
  52. package/src/__tests__/background-workers-disk-pressure.test.ts +1 -0
  53. package/src/__tests__/call-conversation-messages.test.ts +134 -18
  54. package/src/__tests__/call-start-guardian-guard.test.ts +27 -0
  55. package/src/__tests__/channel-approvals.test.ts +0 -27
  56. package/src/__tests__/channel-guardian.test.ts +85 -512
  57. package/src/__tests__/channel-readiness-slack-remote.test.ts +151 -3
  58. package/src/__tests__/channel-retry-sweep.test.ts +88 -1
  59. package/src/__tests__/channel-setup-panel-ack.test.ts +12 -9
  60. package/src/__tests__/chat-reveal-guard-priming.test.ts +31 -34
  61. package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +2 -2
  62. package/src/__tests__/compaction-events.test.ts +8 -8
  63. package/src/__tests__/config-loader-platform-defaults.test.ts +54 -28
  64. package/src/__tests__/config-model-image-provider.test.ts +20 -0
  65. package/src/__tests__/config-schema-memory-substrate.test.ts +306 -0
  66. package/src/__tests__/config-schema.test.ts +14 -2
  67. package/src/__tests__/config-set-platform-guard.test.ts +11 -12
  68. package/src/__tests__/config-sounds-sync.test.ts +4 -4
  69. package/src/__tests__/confirmation-request-guardian-bridge.test.ts +5 -0
  70. package/src/__tests__/context-search-memory-source.test.ts +1 -1
  71. package/src/__tests__/context-search-memory-v2-source.test.ts +6 -6
  72. package/src/__tests__/context-token-estimator.test.ts +54 -1
  73. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +12 -3
  74. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +13 -4
  75. package/src/__tests__/conversation-agent-loop-handlers-max-tokens.test.ts +4 -4
  76. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +10 -1
  77. package/src/__tests__/conversation-agent-loop-overflow.test.ts +22 -13
  78. package/src/__tests__/conversation-agent-loop.test.ts +39 -30
  79. package/src/__tests__/conversation-confirmation-signals.test.ts +14 -14
  80. package/src/__tests__/conversation-evictor.test.ts +49 -0
  81. package/src/__tests__/conversation-fork-crud.test.ts +1 -1
  82. package/src/__tests__/conversation-group-tools.test.ts +279 -0
  83. package/src/__tests__/conversation-history-web-search.test.ts +1 -1
  84. package/src/__tests__/conversation-key-store-disk-view.test.ts +5 -5
  85. package/src/__tests__/conversation-lineage.test.ts +235 -0
  86. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +5 -5
  87. package/src/__tests__/conversation-message-sync-tags.test.ts +4 -4
  88. package/src/__tests__/conversation-queue.test.ts +81 -81
  89. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +8 -1
  90. package/src/__tests__/conversation-routes-hidden-queue.test.ts +8 -1
  91. package/src/__tests__/conversation-routes-slash-commands.test.ts +10 -1
  92. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -2
  93. package/src/__tests__/conversation-slash-queue.test.ts +11 -11
  94. package/src/__tests__/conversation-slash-unknown.test.ts +2 -2
  95. package/src/__tests__/conversation-speed-override.test.ts +2 -2
  96. package/src/__tests__/conversation-summarize-route.test.ts +9 -0
  97. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +18 -16
  98. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +17 -17
  99. package/src/__tests__/conversation-surfaces-app-open.test.ts +7 -4
  100. package/src/__tests__/conversation-surfaces-data-persist.test.ts +7 -7
  101. package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +9 -9
  102. package/src/__tests__/conversation-surfaces-standalone.test.ts +12 -12
  103. package/src/__tests__/conversation-surfaces-state-update.test.ts +8 -5
  104. package/src/__tests__/conversation-surfaces-table-action.test.ts +6 -3
  105. package/src/__tests__/conversation-surfaces-task-progress.test.ts +46 -46
  106. package/src/__tests__/conversation-sync-tags.test.ts +5 -5
  107. package/src/__tests__/conversation-wait-for-idle.test.ts +2 -2
  108. package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -4
  109. package/src/__tests__/conversation-workspace-injection.test.ts +0 -3
  110. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -3
  111. package/src/__tests__/copy-composer-tc-templates.test.ts +3 -131
  112. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  113. package/src/__tests__/daemon-assistant-events.test.ts +14 -15
  114. package/src/__tests__/db-test-helpers.ts +13 -27
  115. package/src/__tests__/disk-pressure-routes.test.ts +12 -3
  116. package/src/__tests__/document-tool-security.test.ts +95 -0
  117. package/src/__tests__/drain-kick-guard.test.ts +232 -0
  118. package/src/__tests__/dynamic-skill-background-guard.test.ts +19 -6
  119. package/src/__tests__/emit-event-signal.test.ts +5 -6
  120. package/src/__tests__/encrypted-store-test-helpers.ts +10 -20
  121. package/src/__tests__/events-tail-route.test.ts +6 -4
  122. package/src/__tests__/feature-flag-test-helpers.ts +10 -20
  123. package/src/__tests__/file-edit-tool.test.ts +24 -4
  124. package/src/__tests__/file-list-tool.test.ts +2 -1
  125. package/src/__tests__/file-read-tool.test.ts +62 -10
  126. package/src/__tests__/file-write-tool.test.ts +24 -5
  127. package/src/__tests__/followup-tools.test.ts +58 -0
  128. package/src/__tests__/gateway-threshold-reader-mock.ts +99 -0
  129. package/src/__tests__/graph-extraction-event-date.test.ts +1 -1
  130. package/src/__tests__/guardian-question-mode.test.ts +42 -3
  131. package/src/__tests__/guardian-reply-router-answer-mode.test.ts +283 -0
  132. package/src/__tests__/guardian-vellum-destination-pinning.test.ts +221 -0
  133. package/src/__tests__/heartbeat-disk-pressure.test.ts +1 -0
  134. package/src/__tests__/heartbeat-service.test.ts +1 -0
  135. package/src/__tests__/helpers/native-web-search-harness.ts +10 -10
  136. package/src/__tests__/host-browser-proxy.test.ts +1 -1
  137. package/src/__tests__/host-file-edit-tool.test.ts +3 -3
  138. package/src/__tests__/host-file-read-tool.test.ts +2 -2
  139. package/src/__tests__/host-file-write-tool.test.ts +7 -3
  140. package/src/__tests__/host-shell-tool.test.ts +17 -5
  141. package/src/__tests__/host-transfer-pending-interactions.test.ts +8 -8
  142. package/src/__tests__/image-credentials.test.ts +65 -8
  143. package/src/__tests__/injection-block.test.ts +4 -4
  144. package/src/__tests__/injector-chain.test.ts +1 -1
  145. package/src/__tests__/injector-pkb-v2-silenced.test.ts +2 -2
  146. package/src/__tests__/injector-registry-order-guard.test.ts +4 -9
  147. package/src/__tests__/injector-v3-suppression.test.ts +6 -4
  148. package/src/__tests__/lifecycle-memory-v2-seed.test.ts +27 -28
  149. package/src/__tests__/list-messages-page-latest.test.ts +2 -2
  150. package/src/__tests__/llm-usage-store.test.ts +67 -4
  151. package/src/__tests__/log-redact.test.ts +18 -0
  152. package/src/__tests__/managed-profile-guard.test.ts +27 -0
  153. package/src/__tests__/managed-skill-lifecycle.test.ts +2 -2
  154. package/src/__tests__/media-generate-image.test.ts +62 -11
  155. package/src/__tests__/memory-identity-context-parity.test.ts +1 -1
  156. package/src/__tests__/memory-jobs-worker-lanes.test.ts +9 -1
  157. package/src/__tests__/memory-retrieval-hook.test.ts +4 -4
  158. package/src/__tests__/notification-decision-strategy.test.ts +115 -11
  159. package/src/__tests__/notification-deep-link.test.ts +12 -12
  160. package/src/__tests__/notification-vellum-adapter.test.ts +8 -8
  161. package/src/__tests__/outbound-slack-persistence.test.ts +8 -8
  162. package/src/__tests__/path-policy.test.ts +315 -0
  163. package/src/__tests__/pending-interactions-resolved-event.test.ts +6 -6
  164. package/src/__tests__/persistence-layering-guard.test.ts +2 -2
  165. package/src/__tests__/pkb-autoinject.test.ts +1 -1
  166. package/src/__tests__/platform-bash-auto-approve.test.ts +1 -0
  167. package/src/__tests__/platform-callback-registration.test.ts +10 -10
  168. package/src/__tests__/plugin-event-hub-facade.test.ts +22 -15
  169. package/src/__tests__/plugin-import-boundary-guard.test.ts +15 -15
  170. package/src/__tests__/processing-flag-persist-failure.test.ts +0 -4
  171. package/src/__tests__/provider-platform-proxy-integration.test.ts +0 -1
  172. package/src/__tests__/rebuild-index-graph-nodes.test.ts +1 -1
  173. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +5 -5
  174. package/src/__tests__/require-fresh-approval.test.ts +38 -38
  175. package/src/__tests__/resolve-app-id.test.ts +32 -0
  176. package/src/__tests__/run-due-schedules.test.ts +72 -1
  177. package/src/__tests__/runtime-events-sse-parity.test.ts +11 -10
  178. package/src/__tests__/runtime-events-sse.test.ts +1 -1
  179. package/src/__tests__/schedule-routes.test.ts +44 -5
  180. package/src/__tests__/schedule-store.test.ts +3 -3
  181. package/src/__tests__/schedule-tools.test.ts +176 -0
  182. package/src/__tests__/secret-prompt-log-hygiene.test.ts +3 -3
  183. package/src/__tests__/secret-prompter-channel-fallback.test.ts +3 -3
  184. package/src/__tests__/secret-response-routing.test.ts +3 -3
  185. package/src/__tests__/secret-routes-platform-proxy.test.ts +4 -7
  186. package/src/__tests__/secret-scanner.test.ts +36 -0
  187. package/src/__tests__/send-endpoint-busy.test.ts +7 -7
  188. package/src/__tests__/slack-required-scopes-mirror.test.ts +81 -0
  189. package/src/__tests__/starter-task-flow.test.ts +26 -3
  190. package/src/__tests__/subagent-call-site-routing.test.ts +2 -2
  191. package/src/__tests__/subagent-disposal.test.ts +40 -11
  192. package/src/__tests__/subagent-fork-notifications.test.ts +4 -4
  193. package/src/__tests__/subagent-fork-prompt-role.test.ts +3 -3
  194. package/src/__tests__/subagent-fork-spawn.test.ts +4 -4
  195. package/src/__tests__/subagent-manager-notify.test.ts +73 -14
  196. package/src/__tests__/subagent-notify-parent.test.ts +48 -4
  197. package/src/__tests__/subagent-spawn-and-await.test.ts +62 -14
  198. package/src/__tests__/subagent-tools.test.ts +64 -0
  199. package/src/__tests__/sync-message-contract.test.ts +3 -3
  200. package/src/__tests__/terminal-tools.test.ts +12 -3
  201. package/src/__tests__/test-preload.ts +9 -0
  202. package/src/__tests__/tool-approval-handler.test.ts +587 -43
  203. package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +7 -4
  204. package/src/__tests__/tool-executor.test.ts +116 -0
  205. package/src/__tests__/tool-grant-request-escalation.test.ts +5 -0
  206. package/src/__tests__/tool-preview-lifecycle.test.ts +32 -29
  207. package/src/__tests__/tool-result-metadata-plumbing.test.ts +5 -5
  208. package/src/__tests__/tool-side-effects-slack-dm.test.ts +1 -0
  209. package/src/__tests__/tool-start-timestamp.test.ts +4 -4
  210. package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +9 -22
  211. package/src/__tests__/ui-choice-copy-surfaces.test.ts +13 -13
  212. package/src/__tests__/ui-work-result-surface.test.ts +3 -3
  213. package/src/__tests__/voice-scoped-grant-consumer.test.ts +6 -6
  214. package/src/__tests__/voice-session-bridge.test.ts +44 -44
  215. package/src/__tests__/workspace-migration-134-image-generation-mode-to-provider.test.ts +284 -0
  216. package/src/__tests__/workspace-policy.test.ts +482 -2
  217. package/src/acp/__tests__/client-handler.test.ts +3 -3
  218. package/src/acp/__tests__/session-manager-persistence.test.ts +10 -10
  219. package/src/acp/__tests__/session-manager-resume.test.ts +7 -7
  220. package/src/acp/client-handler.ts +3 -3
  221. package/src/acp/session-manager.ts +8 -8
  222. package/src/api/events/subagent-event.ts +2 -2
  223. package/src/api/responses/memory-v2-activation-log.ts +2 -2
  224. package/src/api/responses/subagent-detail.ts +13 -0
  225. package/src/approvals/AGENTS.md +2 -0
  226. package/src/approvals/guardian-request-resolvers.ts +118 -24
  227. package/src/apps/app-store.ts +95 -4
  228. package/src/calls/__tests__/voice-control-protocol.test.ts +57 -0
  229. package/src/calls/__tests__/voice-session-bridge.test.ts +229 -7
  230. package/src/calls/__tests__/voice-triage-escalate.test.ts +3 -4
  231. package/src/calls/call-controller.ts +27 -27
  232. package/src/calls/call-conversation-messages.ts +58 -18
  233. package/src/calls/voice-control-protocol.ts +56 -0
  234. package/src/calls/voice-session-bridge.ts +178 -65
  235. package/src/calls/voice-triage-escalate.ts +2 -3
  236. package/src/channels/config.ts +8 -0
  237. package/src/channels/types.ts +1 -0
  238. package/src/cli/commands/config.ts +49 -13
  239. package/src/cli/commands/memory/index.help.ts +4 -2
  240. package/src/cli/commands/memory/memory-retrospective.ts +5 -0
  241. package/src/cli/commands/webhooks.help.ts +9 -4
  242. package/src/cli/lib/bundled-marketplace.json +12 -0
  243. package/src/cli.ts +2 -109
  244. package/src/config/__tests__/deployment-context-defaults.test.ts +9 -0
  245. package/src/config/__tests__/substrate-twin-shadowing.test.ts +191 -0
  246. package/src/config/assistant-feature-flags.ts +1 -1
  247. package/src/config/bundled-skills/conversation-groups/SKILL.md +29 -0
  248. package/src/config/bundled-skills/conversation-groups/TOOLS.json +58 -0
  249. package/src/config/bundled-skills/conversation-groups/tools/group-create.ts +12 -0
  250. package/src/config/bundled-skills/conversation-groups/tools/group-list.ts +12 -0
  251. package/src/config/bundled-skills/conversation-groups/tools/move-to-group.ts +12 -0
  252. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +13 -7
  253. package/src/config/bundled-skills/schedule/SKILL.md +2 -0
  254. package/src/config/feature-flag-cache.ts +6 -16
  255. package/src/config/feature-flag-registry.json +14 -38
  256. package/src/config/loader.ts +7 -1
  257. package/src/config/memory-tier.ts +15 -10
  258. package/src/config/memory-v3-gate.test.ts +278 -1
  259. package/src/config/memory-v3-gate.ts +88 -26
  260. package/src/config/raw-config-utils.ts +1 -1
  261. package/src/config/schemas/__tests__/live-voice.test.ts +27 -16
  262. package/src/config/schemas/__tests__/memory-v3.test.ts +1 -0
  263. package/src/config/schemas/live-voice.ts +33 -7
  264. package/src/config/schemas/llm.ts +1 -0
  265. package/src/config/schemas/memory-retrospective.ts +13 -0
  266. package/src/config/schemas/memory-substrate.ts +183 -0
  267. package/src/config/schemas/memory-v2.ts +3 -2
  268. package/src/config/schemas/memory-v3.ts +11 -4
  269. package/src/config/schemas/memory.ts +35 -1
  270. package/src/config/schemas/services.ts +10 -2
  271. package/src/config/substrate-twin-shadowing.ts +195 -0
  272. package/src/config/webhook-routing.ts +67 -0
  273. package/src/context/compactor.ts +10 -3
  274. package/src/context/token-estimator.ts +21 -0
  275. package/src/daemon/__tests__/channel-ui-capability.test.ts +85 -0
  276. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +3 -3
  277. package/src/daemon/__tests__/embedding-reconcile.test.ts +3 -3
  278. package/src/daemon/__tests__/inference-profile-notification.test.ts +3 -6
  279. package/src/daemon/__tests__/native-web-search-metadata.test.ts +5 -5
  280. package/src/daemon/__tests__/web-search-status-text.test.ts +2 -2
  281. package/src/daemon/channel-ui-capability.ts +70 -1
  282. package/src/daemon/conversation-agent-loop-handlers.ts +4 -4
  283. package/src/daemon/conversation-agent-loop.ts +12 -5
  284. package/src/daemon/conversation-evictor.ts +26 -4
  285. package/src/daemon/conversation-launch.ts +11 -14
  286. package/src/daemon/conversation-lineage.ts +76 -0
  287. package/src/daemon/conversation-messaging.ts +14 -14
  288. package/src/daemon/conversation-notifiers.ts +2 -2
  289. package/src/daemon/conversation-process.ts +133 -21
  290. package/src/daemon/conversation-queue-manager.ts +7 -7
  291. package/src/daemon/conversation-runtime-assembly.ts +16 -0
  292. package/src/daemon/conversation-store.ts +11 -3
  293. package/src/daemon/conversation-surfaces.ts +8 -14
  294. package/src/daemon/conversation-tool-setup.ts +13 -7
  295. package/src/daemon/conversation-usage.ts +12 -11
  296. package/src/daemon/conversation.ts +38 -15
  297. package/src/daemon/disk-pressure-guard.ts +5 -12
  298. package/src/daemon/embedding-reconcile.ts +14 -9
  299. package/src/daemon/handlers/config-channels.ts +40 -134
  300. package/src/daemon/handlers/config-model.ts +15 -7
  301. package/src/daemon/handlers/conversations.ts +4 -2
  302. package/src/daemon/handlers/shared.ts +29 -0
  303. package/src/daemon/host-proxy-base.ts +4 -4
  304. package/src/daemon/lifecycle.ts +13 -0
  305. package/src/daemon/message-protocol.ts +8 -144
  306. package/src/daemon/message-types/computer-use.ts +0 -12
  307. package/src/daemon/message-types/conversations.ts +9 -345
  308. package/src/daemon/message-types/host-app-control.ts +0 -9
  309. package/src/daemon/message-types/host-browser.ts +0 -7
  310. package/src/daemon/message-types/host-ui-snapshot.ts +0 -7
  311. package/src/daemon/message-types/messages.ts +0 -48
  312. package/src/daemon/message-types/notifications.ts +0 -7
  313. package/src/daemon/message-types/skills.ts +0 -5
  314. package/src/daemon/message-types/surfaces.ts +0 -9
  315. package/src/daemon/message-types/sync.ts +0 -2
  316. package/src/daemon/process-message.ts +5 -5
  317. package/src/daemon/skill-memory-refresh.ts +5 -5
  318. package/src/daemon/tool-side-effects.ts +32 -16
  319. package/src/daemon/trust-context.ts +1 -1
  320. package/src/daemon/wake-conversation-ops.ts +4 -4
  321. package/src/heartbeat/__tests__/heartbeat-run-store.test.ts +100 -7
  322. package/src/heartbeat/__tests__/heartbeat-service.test.ts +67 -0
  323. package/src/heartbeat/heartbeat-run-store.ts +81 -4
  324. package/src/heartbeat/heartbeat-service.ts +32 -1
  325. package/src/home/__tests__/feed-writer.test.ts +7 -7
  326. package/src/home/__tests__/home-content-refresh.test.ts +3 -0
  327. package/src/home/__tests__/suggested-prompts.test.ts +1 -0
  328. package/src/home/feed-writer.ts +6 -13
  329. package/src/home/home-content-refresh.ts +6 -9
  330. package/src/home/relationship-state-writer.ts +5 -12
  331. package/src/home/suggested-prompts-cache.ts +6 -19
  332. package/src/ipc/__tests__/events-publish-client.test.ts +67 -0
  333. package/src/ipc/events-publish-client.ts +44 -0
  334. package/src/ipc/routes/__tests__/conversation-sync-ipc-routes.test.ts +2 -2
  335. package/src/ipc/routes/__tests__/events-ipc-routes.test.ts +38 -9
  336. package/src/ipc/routes/events-ipc-routes.ts +11 -31
  337. package/src/live-voice/__tests__/front-decision.test.ts +4 -176
  338. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +276 -68
  339. package/src/live-voice/__tests__/live-voice-continuation-fork-hydration.test.ts +286 -0
  340. package/src/live-voice/__tests__/live-voice-events.test.ts +6 -3
  341. package/src/live-voice/__tests__/live-voice-progress.test.ts +185 -46
  342. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +24 -59
  343. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +38 -54
  344. package/src/live-voice/__tests__/live-voice-vad.test.ts +1154 -636
  345. package/src/live-voice/front-decision.ts +5 -155
  346. package/src/live-voice/live-voice-metrics.ts +4 -1
  347. package/src/live-voice/live-voice-session.ts +1288 -439
  348. package/src/live-voice/progress-phrases.ts +17 -0
  349. package/src/live-voice/protocol.ts +15 -0
  350. package/src/media/app-icon-generator.ts +15 -8
  351. package/src/media/avatar-router.ts +9 -5
  352. package/src/media/image-credentials.ts +42 -10
  353. package/src/media/image-service.ts +1 -13
  354. package/src/media/types.ts +26 -3
  355. package/src/notifications/AGENTS.md +2 -0
  356. package/src/notifications/__tests__/broadcaster.test.ts +95 -0
  357. package/src/notifications/adapters/macos.ts +3 -3
  358. package/src/notifications/broadcaster.ts +57 -0
  359. package/src/notifications/copy-composer.ts +0 -15
  360. package/src/notifications/decision-engine.ts +21 -22
  361. package/src/notifications/guardian-question-mode.ts +57 -2
  362. package/src/notifications/home-feed-side-effect.ts +4 -0
  363. package/src/notifications/signal.ts +6 -4
  364. package/src/notifications/trusted-contact-payloads.ts +3 -4
  365. package/src/notifications/vellum-card-affinity.ts +39 -0
  366. package/src/oauth/AGENTS.md +1 -1
  367. package/src/permissions/channel-permission-query.test.ts +69 -0
  368. package/src/permissions/channel-permission-query.ts +73 -1
  369. package/src/permissions/checker.test.ts +298 -15
  370. package/src/permissions/checker.ts +102 -36
  371. package/src/permissions/confirmation-guardian-request.test.ts +2 -2
  372. package/src/permissions/confirmation-guardian-request.ts +2 -2
  373. package/src/permissions/gateway-threshold-reader.test.ts +224 -17
  374. package/src/permissions/gateway-threshold-reader.ts +82 -10
  375. package/src/permissions/ipc-risk-types.ts +8 -0
  376. package/src/permissions/prompter.ts +5 -5
  377. package/src/permissions/question-guardian-request.test.ts +222 -0
  378. package/src/permissions/question-guardian-request.ts +179 -0
  379. package/src/permissions/question-prompter.test.ts +8 -8
  380. package/src/permissions/question-prompter.ts +18 -0
  381. package/src/permissions/secret-prompter.ts +2 -4
  382. package/src/permissions/workspace-policy.ts +269 -34
  383. package/src/persistence/__tests__/lifecycle-quiesce.test.ts +82 -0
  384. package/src/persistence/__tests__/message-content-file.test.ts +58 -3
  385. package/src/persistence/conversation-crud.ts +93 -0
  386. package/src/persistence/conversation-key-store.ts +63 -0
  387. package/src/persistence/conversation-parent.ts +27 -0
  388. package/src/persistence/db-async-query.ts +1 -1
  389. package/src/persistence/db-singleton.ts +14 -27
  390. package/src/persistence/delivery-crud.ts +45 -0
  391. package/src/persistence/embeddings/embedding-backend.ts +2 -1
  392. package/src/persistence/embeddings/embedding-identity.ts +7 -3
  393. package/src/persistence/job-utils.ts +32 -27
  394. package/src/persistence/jobs-store.ts +92 -3
  395. package/src/persistence/lifecycle-quiesce.ts +80 -0
  396. package/src/persistence/llm-usage-store.ts +45 -8
  397. package/src/persistence/message-content-file.ts +4 -4
  398. package/src/persistence/migrations/336-move-memory-v2-activation-logs-to-memory-db.ts +1 -1
  399. package/src/persistence/migrations/351-schedule-skill-script-handoff.ts +39 -0
  400. package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.test.ts +83 -0
  401. package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.ts +36 -0
  402. package/src/persistence/migrations/353-add-llm-usage-conversation-type.test.ts +110 -0
  403. package/src/persistence/migrations/353-add-llm-usage-conversation-type.ts +49 -0
  404. package/src/persistence/migrations/354-backfill-app-conversation-lineage.ts +41 -0
  405. package/src/persistence/schema/infrastructure.ts +11 -0
  406. package/src/persistence/steps.ts +9 -1
  407. package/src/plugin-api/__tests__/publish-event-forwarding.test.ts +113 -0
  408. package/src/plugin-api/__tests__/publish-event.test.ts +57 -0
  409. package/src/plugin-api/conversation-turn.ts +4 -4
  410. package/src/plugin-api/event-hub-facade.ts +15 -6
  411. package/src/plugin-api/index.ts +10 -4
  412. package/src/plugin-api/publish-event.ts +36 -0
  413. package/src/plugins/AGENTS.md +2 -0
  414. package/src/plugins/defaults/compaction/window-manager.ts +9 -0
  415. package/src/plugins/defaults/index.ts +9 -13
  416. package/src/plugins/defaults/memory/AGENTS.md +775 -0
  417. package/src/plugins/defaults/memory/__tests__/capability-seed-embed-gate.test.ts +186 -0
  418. package/src/plugins/defaults/memory/__tests__/fixtures/memory-v2-activation-fixtures.ts +1 -1
  419. package/src/plugins/defaults/memory/__tests__/indexer-tier-dispatch.test.ts +129 -0
  420. package/src/plugins/defaults/memory/__tests__/jobs-worker-quiesce-gate.test.ts +91 -0
  421. package/src/plugins/defaults/memory/__tests__/jobs-worker-v1-graph-substrate-noop.test.ts +342 -0
  422. package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-graph-trigger-embed.test.ts +9 -1
  423. package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-schedule.test.ts +36 -1
  424. package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-sweep-defer.test.ts +9 -1
  425. package/src/plugins/defaults/memory/__tests__/memory-log-stores-degraded.test.ts +2 -2
  426. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +44 -10
  427. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +151 -10
  428. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1155 -0
  429. package/src/plugins/defaults/memory/__tests__/memory-v2-activation-log-store.test.ts +1 -1
  430. package/src/plugins/defaults/memory/__tests__/memory-v2-concept-frequency.test.ts +3 -3
  431. package/src/plugins/defaults/memory/__tests__/tools-delete-memory-page.test.ts +1 -1
  432. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +71 -35
  433. package/src/plugins/defaults/memory/context-search/sources/memory.ts +5 -1
  434. package/src/plugins/defaults/memory/fork-conversation-memory.ts +4 -4
  435. package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +80 -10
  436. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +1 -1
  437. package/src/plugins/defaults/memory/graph/capability-seed.ts +24 -10
  438. package/src/plugins/defaults/memory/graph/conversation-graph-memory.ts +123 -49
  439. package/src/plugins/defaults/memory/graph/graph-search.ts +6 -118
  440. package/src/plugins/defaults/memory/graph/in-context-tracker.test.ts +173 -0
  441. package/src/plugins/defaults/memory/graph/in-context-tracker.ts +109 -0
  442. package/src/plugins/defaults/memory/graph/store.ts +1 -0
  443. package/src/plugins/defaults/memory/graph/tool-handlers.test.ts +1 -1
  444. package/src/plugins/defaults/memory/graph/tool-handlers.ts +5 -2
  445. package/src/plugins/defaults/memory/graph-topology/__tests__/build-memory-graph.test.ts +1 -1
  446. package/src/plugins/defaults/memory/graph-topology/build-memory-graph.ts +6 -9
  447. package/src/plugins/defaults/memory/hooks/__tests__/user-prompt-submit.test.ts +30 -16
  448. package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts +18 -14
  449. package/src/plugins/defaults/memory/indexer.ts +146 -89
  450. package/src/plugins/defaults/memory/injectors.ts +86 -41
  451. package/src/plugins/defaults/memory/job-handlers.ts +130 -39
  452. package/src/plugins/defaults/memory/jobs/__tests__/embed-concept-page.test.ts +2 -2
  453. package/src/plugins/defaults/memory/jobs/embed-concept-page.ts +22 -10
  454. package/src/plugins/defaults/memory/jobs-worker.ts +279 -208
  455. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +4 -2
  456. package/src/plugins/defaults/memory/memory-retrospective-job.ts +40 -3
  457. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +99 -11
  458. package/src/plugins/defaults/memory/src/__tests__/memory-v2-routes.test.ts +5 -3
  459. package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +3 -4
  460. package/src/plugins/defaults/memory/src/memory-graph-routes.ts +2 -1
  461. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +7 -5
  462. package/src/plugins/defaults/memory/src/memory-item-routes.ts +51 -19
  463. package/src/plugins/defaults/memory/src/memory-v2-routes.ts +31 -20
  464. package/src/plugins/defaults/memory/startup.ts +58 -21
  465. package/src/plugins/defaults/memory/substrate/__tests__/boot-maintenance.test.ts +335 -0
  466. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/cli-command-store.test.ts +14 -8
  467. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-job.test.ts +51 -21
  468. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/edge-index.test.ts +1 -1
  469. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/frontmatter-sweep.test.ts +3 -3
  470. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-index.test.ts +1 -1
  471. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-store.test.ts +1 -1
  472. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/prompts-consolidation.test.ts +2 -2
  473. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/qdrant.test.ts +23 -14
  474. package/src/plugins/defaults/memory/substrate/__tests__/reembed-job.test.ts +262 -0
  475. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sim.test.ts +9 -12
  476. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-content.test.ts +2 -2
  477. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-store.test.ts +9 -9
  478. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sparse-bm25.test.ts +12 -5
  479. package/src/plugins/defaults/memory/substrate/__tests__/spread.test.ts +276 -0
  480. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/static-context.test.ts +17 -8
  481. package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sweep-job.test.ts +9 -9
  482. package/src/plugins/defaults/memory/{v3/substrate/memory-v2-startup.ts → substrate/boot-maintenance.ts} +43 -24
  483. package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-store.ts +13 -8
  484. package/src/plugins/defaults/memory/{v3/substrate → substrate}/consolidation-job.ts +51 -27
  485. package/src/plugins/defaults/memory/{v3/substrate → substrate}/constants.ts +1 -0
  486. package/src/plugins/defaults/memory/{v3/substrate → substrate}/edge-index.ts +28 -9
  487. package/src/plugins/defaults/memory/{v3/substrate → substrate}/frontmatter-sweep.ts +14 -7
  488. package/src/plugins/defaults/memory/{v3/substrate → substrate}/injected-block-slugs.ts +9 -3
  489. package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-index.ts +29 -10
  490. package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-store.ts +35 -12
  491. package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/consolidation.ts +14 -8
  492. package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/sweep.ts +4 -1
  493. package/src/plugins/defaults/memory/{v3/substrate → substrate}/qdrant.ts +96 -36
  494. package/src/plugins/defaults/memory/substrate/reembed-job.ts +54 -0
  495. package/src/plugins/defaults/memory/{v3/substrate → substrate}/sim.ts +26 -15
  496. package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-content.ts +1 -1
  497. package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-store.ts +13 -8
  498. package/src/plugins/defaults/memory/{v3/substrate → substrate}/sparse-bm25.ts +27 -10
  499. package/src/plugins/defaults/memory/substrate/spread.ts +165 -0
  500. package/src/plugins/defaults/memory/{v3/substrate → substrate}/static-context.ts +12 -7
  501. package/src/plugins/defaults/memory/{v3/substrate → substrate}/sweep-job.ts +20 -15
  502. package/src/plugins/defaults/memory/substrate/tuning.ts +93 -0
  503. package/src/plugins/defaults/memory/{v3/substrate → substrate}/types.ts +7 -4
  504. package/src/plugins/defaults/memory/tools.test.ts +1 -1
  505. package/src/plugins/defaults/memory/tools.ts +1 -1
  506. package/src/plugins/defaults/memory/v1/README.md +5 -0
  507. package/src/plugins/defaults/memory/{__tests__ → v1/__tests__}/filing-jobs.test.ts +4 -4
  508. package/src/plugins/defaults/memory/{filing-jobs.ts → v1/filing-jobs.ts} +3 -3
  509. package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.test.ts +68 -6
  510. package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.ts +40 -21
  511. package/src/plugins/defaults/memory/{graph → v1/graph}/consolidation.ts +14 -14
  512. package/src/plugins/defaults/memory/{graph → v1/graph}/decay.test.ts +1 -1
  513. package/src/plugins/defaults/memory/{graph → v1/graph}/decay.ts +3 -3
  514. package/src/plugins/defaults/memory/{graph → v1/graph}/extraction-job.ts +5 -5
  515. package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.test.ts +1 -1
  516. package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.ts +20 -14
  517. package/src/plugins/defaults/memory/{graph → v1/graph}/graph-search.test.ts +3 -3
  518. package/src/plugins/defaults/memory/v1/graph/graph-search.ts +123 -0
  519. package/src/plugins/defaults/memory/{graph → v1/graph}/injection.test.ts +1 -118
  520. package/src/plugins/defaults/memory/{graph → v1/graph}/injection.ts +9 -111
  521. package/src/plugins/defaults/memory/{graph → v1/graph}/narrative.ts +6 -6
  522. package/src/plugins/defaults/memory/{graph → v1/graph}/pattern-scan.ts +5 -5
  523. package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.test.ts +135 -42
  524. package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.ts +40 -22
  525. package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.test.ts +1 -1
  526. package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.ts +1 -1
  527. package/src/plugins/defaults/memory/{graph → v1/graph}/serendipity.ts +1 -1
  528. package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.test.ts +1 -1
  529. package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.ts +1 -1
  530. package/src/plugins/defaults/memory/{identity-context.ts → v1/identity-context.ts} +4 -4
  531. package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/backfill.ts +6 -6
  532. package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.test.ts +6 -6
  533. package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.ts +6 -6
  534. package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/index-maintenance.ts +8 -8
  535. package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.test.ts +7 -7
  536. package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.ts +5 -5
  537. package/src/plugins/defaults/memory/{pkb → v1/pkb}/autoinject.ts +1 -1
  538. package/src/plugins/defaults/memory/{pkb → v1/pkb}/context.ts +2 -2
  539. package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.test.ts +3 -3
  540. package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.ts +2 -2
  541. package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.test.ts +3 -3
  542. package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.ts +2 -2
  543. package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.test.ts +3 -3
  544. package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.ts +6 -6
  545. package/src/plugins/defaults/memory/{pkb → v1/pkb}/types.ts +3 -5
  546. package/src/plugins/defaults/memory/{pkb-schedule.ts → v1/pkb-schedule.ts} +2 -2
  547. package/src/plugins/defaults/memory/{search/semantic.ts → v1/semantic-search.ts} +1 -1
  548. package/src/plugins/defaults/memory/v2/__tests__/activation-store.test.ts +1 -1
  549. package/src/plugins/defaults/memory/v2/__tests__/activation.test.ts +11 -274
  550. package/src/plugins/defaults/memory/v2/__tests__/backfill-jobs.test.ts +9 -131
  551. package/src/plugins/defaults/memory/v2/__tests__/harness-compare.test.ts +1 -1
  552. package/src/plugins/defaults/memory/v2/__tests__/harness-oracle.test.ts +1 -1
  553. package/src/plugins/defaults/memory/v2/__tests__/harness-replay-input.test.ts +1 -1
  554. package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +14 -10
  555. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +7 -5
  556. package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +12 -6
  557. package/src/plugins/defaults/memory/{memory-v2-activation-log-store.ts → v2/activation-log-store.ts} +2 -2
  558. package/src/plugins/defaults/memory/v2/activation-store.ts +22 -8
  559. package/src/plugins/defaults/memory/v2/activation.ts +30 -163
  560. package/src/plugins/defaults/memory/v2/backfill-jobs.ts +7 -54
  561. package/src/plugins/defaults/memory/{memory-v2-concept-frequency.ts → v2/concept-frequency.ts} +3 -3
  562. package/src/plugins/defaults/memory/v2/harness/oracle.ts +1 -1
  563. package/src/plugins/defaults/memory/v2/harness/replay-input.ts +2 -2
  564. package/src/plugins/defaults/memory/v2/harness/retriever.ts +1 -1
  565. package/src/plugins/defaults/memory/v2/injection.ts +36 -26
  566. package/src/plugins/defaults/memory/v2/migration.ts +2 -2
  567. package/src/plugins/defaults/memory/{rerank-local.ts → v2/rerank-local.ts} +44 -18
  568. package/src/plugins/defaults/memory/v2/reranker.ts +29 -11
  569. package/src/plugins/defaults/memory/v2/router.ts +38 -15
  570. package/src/plugins/defaults/memory/v3/__tests__/candidate-match.test.ts +5 -3
  571. package/src/plugins/defaults/memory/v3/__tests__/carry-integration.test.ts +21 -8
  572. package/src/plugins/defaults/memory/v3/__tests__/edge.test.ts +1 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/live-integration.test.ts +1 -1
  574. package/src/plugins/defaults/memory/v3/__tests__/maintain-job.test.ts +4 -1
  575. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +181 -3
  576. package/src/plugins/defaults/memory/v3/__tests__/selection-log-store.test.ts +3 -0
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-integration.test.ts +14 -5
  578. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +40 -50
  579. package/src/plugins/defaults/memory/v3/__tests__/tuning-profile.test.ts +1 -0
  580. package/src/plugins/defaults/memory/v3/candidate-match.ts +2 -2
  581. package/src/plugins/defaults/memory/v3/capabilities.ts +6 -4
  582. package/src/plugins/defaults/memory/v3/card.ts +25 -9
  583. package/src/plugins/defaults/memory/v3/edge.ts +52 -18
  584. package/src/plugins/defaults/memory/v3/gate.ts +4 -4
  585. package/src/plugins/defaults/memory/v3/injector.ts +10 -3
  586. package/src/plugins/defaults/memory/v3/maintain-job.ts +3 -3
  587. package/src/plugins/defaults/memory/v3/orchestrate.ts +100 -18
  588. package/src/plugins/defaults/memory/v3/page-content.ts +26 -10
  589. package/src/plugins/defaults/memory/v3/prune.ts +40 -16
  590. package/src/plugins/defaults/memory/v3/selection-log-store.ts +34 -12
  591. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +9 -13
  592. package/src/plugins/defaults/memory/v3/span-query.test.ts +86 -0
  593. package/src/plugins/defaults/memory/v3/span-query.ts +71 -0
  594. package/src/plugins/defaults/memory/v3/tuning-profile.ts +3 -0
  595. package/src/plugins/defaults/memory/v3/types.ts +4 -1
  596. package/src/plugins/defaults/memory/v3-eval/eval-packets.ts +2 -2
  597. package/src/plugins/defaults/memory/validation.ts +1 -1
  598. package/src/plugins/defaults/memory/worker.ts +2 -1
  599. package/src/plugins/defaults/platform-hosted/__tests__/reengage.test.ts +104 -12
  600. package/src/plugins/defaults/platform-hosted/routes/reengage.ts +104 -12
  601. package/src/plugins/types.ts +2 -2
  602. package/src/prompts/sections.ts +4 -16
  603. package/src/providers/__tests__/registry-native-web-search.test.ts +47 -2
  604. package/src/providers/anthropic/client.ts +15 -6
  605. package/src/providers/content-block-schema.ts +26 -0
  606. package/src/providers/inference/__tests__/connections-credential-normalize.test.ts +87 -0
  607. package/src/providers/inference/adapter-factory.ts +6 -0
  608. package/src/providers/inference/connections.ts +30 -14
  609. package/src/providers/model-catalog.ts +77 -0
  610. package/src/providers/openai/__tests__/chat-completions-input-audio.test.ts +260 -0
  611. package/src/providers/openai/chat-completions-provider.ts +65 -17
  612. package/src/providers/openai/input-audio.ts +76 -0
  613. package/src/providers/openrouter/client.ts +0 -2
  614. package/src/providers/platform-proxy/constants.ts +4 -0
  615. package/src/providers/poolside/client.ts +42 -0
  616. package/src/providers/registry.ts +40 -8
  617. package/src/providers/retry.ts +2 -0
  618. package/src/providers/types.ts +39 -1
  619. package/src/providers/vercel-ai-gateway/client.ts +0 -2
  620. package/src/runtime/AGENTS.md +7 -5
  621. package/src/runtime/__tests__/question-request-guardian-bridge.test.ts +173 -0
  622. package/src/runtime/__tests__/question-resolution.test.ts +111 -0
  623. package/src/runtime/access-request-helper.ts +7 -15
  624. package/src/runtime/assistant-event-hub.ts +43 -31
  625. package/src/runtime/assistant-event-publish-options.ts +31 -0
  626. package/src/runtime/assistant-event.ts +19 -30
  627. package/src/runtime/assistant-stream-state.ts +4 -4
  628. package/src/runtime/channel-approvals.ts +0 -25
  629. package/src/runtime/channel-readiness-service.ts +99 -40
  630. package/src/runtime/channel-retry-sweep.ts +38 -15
  631. package/src/runtime/confirmation-request-guardian-bridge.ts +4 -0
  632. package/src/runtime/guardian-reply-router.ts +108 -0
  633. package/src/runtime/http-server.ts +8 -0
  634. package/src/runtime/http-types.ts +2 -2
  635. package/src/runtime/process-role.ts +42 -0
  636. package/src/runtime/question-request-guardian-bridge.ts +184 -0
  637. package/src/runtime/question-resolution.ts +153 -0
  638. package/src/runtime/routes/__tests__/bookmark-routes.test.ts +6 -2
  639. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +1 -1
  640. package/src/runtime/routes/__tests__/subagents-routes.test.ts +250 -0
  641. package/src/runtime/routes/__tests__/webhook-routes.test.ts +201 -0
  642. package/src/runtime/routes/acp-routes.ts +2 -2
  643. package/src/runtime/routes/bookmark-routes.ts +3 -15
  644. package/src/runtime/routes/canned-message-complete.ts +2 -2
  645. package/src/runtime/routes/consolidation-routes.ts +8 -2
  646. package/src/runtime/routes/conversation-list-routes.ts +3 -19
  647. package/src/runtime/routes/conversation-management-routes.ts +1 -5
  648. package/src/runtime/routes/conversation-query-routes.ts +52 -8
  649. package/src/runtime/routes/conversation-routes.ts +36 -22
  650. package/src/runtime/routes/credential-request-routes.ts +0 -1
  651. package/src/runtime/routes/filing-routes.ts +7 -7
  652. package/src/runtime/routes/global-search-routes.ts +1 -1
  653. package/src/runtime/routes/guardian-approval-prompt.ts +2 -2
  654. package/src/runtime/routes/image-generation-routes.ts +12 -9
  655. package/src/runtime/routes/inbound-message-handler.ts +17 -0
  656. package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +50 -0
  657. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +19 -0
  658. package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -4
  659. package/src/runtime/routes/inbound-stages/inbound-content-prep.test.ts +129 -0
  660. package/src/runtime/routes/inbound-stages/inbound-content-prep.ts +141 -9
  661. package/src/runtime/routes/index.ts +2 -0
  662. package/src/runtime/routes/lifecycle-routes.ts +199 -0
  663. package/src/runtime/routes/platform-routes.ts +3 -8
  664. package/src/runtime/routes/playground/__tests__/inject-failures.test.ts +7 -7
  665. package/src/runtime/routes/playground/__tests__/reset-circuit.test.ts +5 -5
  666. package/src/runtime/routes/question-routes.ts +34 -87
  667. package/src/runtime/routes/secret-routes.ts +11 -1
  668. package/src/runtime/routes/settings-routes.ts +8 -18
  669. package/src/runtime/routes/subagents-routes.ts +35 -8
  670. package/src/runtime/routes/upgrade-broadcast-routes.ts +4 -5
  671. package/src/runtime/routes/webhook-routes.ts +89 -30
  672. package/src/runtime/routes/workspace-routes.test.ts +7 -4
  673. package/src/runtime/slack-reply-session.test.ts +12 -12
  674. package/src/runtime/slack-reply-session.ts +17 -17
  675. package/src/runtime/sync/sync-publisher.test.ts +7 -5
  676. package/src/runtime/tool-grant-request-helper.ts +4 -0
  677. package/src/schedule/schedule-store.ts +70 -0
  678. package/src/schedule/scheduler.ts +27 -0
  679. package/src/security/AGENTS.md +1 -1
  680. package/src/security/credential-key.ts +17 -0
  681. package/src/security/secret-ingress.ts +14 -105
  682. package/src/security/secret-patterns.ts +8 -139
  683. package/src/security/secret-scanner.ts +8 -3
  684. package/src/security/store-path-override.ts +6 -16
  685. package/src/sequence/store.ts +8 -0
  686. package/src/signals/emit-event.ts +5 -10
  687. package/src/signals/event-stream.ts +20 -10
  688. package/src/subagent/manager.ts +76 -25
  689. package/src/subagent/notify.ts +8 -0
  690. package/src/subagent/types.ts +13 -3
  691. package/src/telegram/__tests__/webhook-health.test.ts +442 -0
  692. package/src/telegram/webhook-health.ts +478 -0
  693. package/src/telemetry/AGENTS.md +6 -0
  694. package/src/telemetry/memory-corpus-size.test.ts +129 -0
  695. package/src/telemetry/memory-corpus-size.ts +184 -0
  696. package/src/telemetry/memory-tier-reporter.test.ts +67 -2
  697. package/src/telemetry/memory-tier-reporter.ts +30 -3
  698. package/src/tools/__tests__/conversation-group-tool-input-schemas.test.ts +61 -0
  699. package/src/tools/__tests__/document-tool-input-schemas.test.ts +103 -0
  700. package/src/tools/__tests__/followup-call-acp-tool-input-schemas.test.ts +125 -0
  701. package/src/tools/__tests__/schedule-tool-input-schemas.test.ts +83 -0
  702. package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +116 -0
  703. package/src/tools/__tests__/tool-input-schemas.test.ts +192 -0
  704. package/src/tools/__tests__/tool-schema-drift-harness.ts +142 -0
  705. package/src/tools/acp/abort.ts +21 -1
  706. package/src/tools/acp/context.ts +5 -5
  707. package/src/tools/acp/spawn.test.ts +15 -5
  708. package/src/tools/acp/spawn.ts +26 -3
  709. package/src/tools/acp/status.ts +20 -1
  710. package/src/tools/acp/steer.test.ts +20 -0
  711. package/src/tools/acp/steer.ts +23 -2
  712. package/src/tools/apps/resolve-app-id.ts +15 -3
  713. package/src/tools/ask-question/ask-question-tool.test.ts +65 -0
  714. package/src/tools/ask-question/ask-question-tool.ts +68 -92
  715. package/src/tools/browser/browser-screencast.ts +6 -6
  716. package/src/tools/calls/call-end.ts +24 -3
  717. package/src/tools/calls/call-start.ts +35 -11
  718. package/src/tools/calls/call-status.ts +20 -1
  719. package/src/tools/conversation-groups/group_create.ts +63 -0
  720. package/src/tools/conversation-groups/group_list.ts +20 -0
  721. package/src/tools/conversation-groups/group_shared.ts +52 -0
  722. package/src/tools/conversation-groups/move_to_group.ts +106 -0
  723. package/src/tools/document/document-comment-tool.test.ts +49 -0
  724. package/src/tools/document/document-comment-tool.ts +45 -6
  725. package/src/tools/document/document-tool.ts +142 -24
  726. package/src/tools/executor.ts +27 -13
  727. package/src/tools/filesystem/edit.ts +62 -60
  728. package/src/tools/filesystem/list.ts +36 -29
  729. package/src/tools/filesystem/read.ts +56 -39
  730. package/src/tools/filesystem/write.ts +44 -38
  731. package/src/tools/followups/followup_create.ts +41 -12
  732. package/src/tools/followups/followup_list.ts +34 -6
  733. package/src/tools/followups/followup_resolve.ts +26 -4
  734. package/src/tools/host-filesystem/edit.ts +50 -60
  735. package/src/tools/host-filesystem/read.ts +45 -39
  736. package/src/tools/host-filesystem/write.ts +31 -36
  737. package/src/tools/host-terminal/host-shell.ts +76 -62
  738. package/src/tools/policy-context.ts +27 -10
  739. package/src/tools/schedule/create.ts +76 -33
  740. package/src/tools/schedule/delete.ts +21 -2
  741. package/src/tools/schedule/list.ts +25 -3
  742. package/src/tools/schedule/update.ts +105 -48
  743. package/src/tools/shared/filesystem/path-policy.ts +246 -40
  744. package/src/tools/shared/zod-tool-schema.ts +117 -0
  745. package/src/tools/subagent/abort.ts +12 -4
  746. package/src/tools/subagent/consult-deadline.ts +9 -3
  747. package/src/tools/subagent/message.ts +21 -5
  748. package/src/tools/subagent/notify-parent.ts +44 -23
  749. package/src/tools/subagent/read.ts +15 -4
  750. package/src/tools/subagent/resolve.ts +26 -4
  751. package/src/tools/subagent/spawn.ts +37 -13
  752. package/src/tools/subagent/status.ts +12 -4
  753. package/src/tools/system/request-permission.ts +35 -26
  754. package/src/tools/terminal/shell.ts +68 -55
  755. package/src/tools/tool-approval-handler.ts +327 -53
  756. package/src/tools/tool-input-schemas.ts +85 -0
  757. package/src/tools/tool-types.ts +7 -0
  758. package/src/tools/types.ts +8 -0
  759. package/src/tools/workflows/manage-workflows.ts +30 -2
  760. package/src/tools/workflows/run-workflow.test.ts +54 -2
  761. package/src/tools/workflows/run-workflow.ts +32 -4
  762. package/src/util/fs-symlinks.ts +30 -0
  763. package/src/util/log-redact.ts +7 -3
  764. package/src/util/platform.ts +15 -0
  765. package/src/vellum-assistant-namespace.d.ts +80 -0
  766. package/src/watcher/watcher-store.ts +8 -0
  767. package/src/workflows/leaf-runner.test.ts +104 -2
  768. package/src/workflows/leaf-runner.ts +31 -6
  769. package/src/workspace/migrations/070-memory-v2-summary-schema-rebuild.ts +2 -2
  770. package/src/workspace/migrations/134-image-generation-mode-to-provider.ts +128 -0
  771. package/src/workspace/migrations/135-copy-substrate-tunables.ts +248 -0
  772. package/src/workspace/migrations/__tests__/135-copy-substrate-tunables.test.ts +446 -0
  773. package/src/workspace/migrations/__tests__/backfill-v3-leaves.test.ts +2 -2
  774. package/src/workspace/migrations/registry.ts +4 -0
  775. package/tsconfig.plugin-api.json +1 -1
  776. package/src/__tests__/memory-retrieval-budget.test.ts +0 -49
  777. package/src/__tests__/secret-patterns.test.ts +0 -27
  778. package/src/config/__tests__/proc-to-skills.test.ts +0 -21
  779. package/src/config/memory-tier.test.ts +0 -44
  780. package/src/daemon/message-types/acp.ts +0 -20
  781. package/src/daemon/message-types/apps.ts +0 -11
  782. package/src/daemon/message-types/background-tools.ts +0 -14
  783. package/src/daemon/message-types/bookmarks.ts +0 -13
  784. package/src/daemon/message-types/contacts.ts +0 -13
  785. package/src/daemon/message-types/document-comments.ts +0 -14
  786. package/src/daemon/message-types/documents.ts +0 -13
  787. package/src/daemon/message-types/home.ts +0 -15
  788. package/src/daemon/message-types/host-bash.ts +0 -12
  789. package/src/daemon/message-types/host-cu.ts +0 -10
  790. package/src/daemon/message-types/host-file.ts +0 -12
  791. package/src/daemon/message-types/host-transfer.ts +0 -12
  792. package/src/daemon/message-types/integrations.ts +0 -90
  793. package/src/daemon/message-types/memory.ts +0 -10
  794. package/src/daemon/message-types/schedules.ts +0 -14
  795. package/src/daemon/message-types/settings.ts +0 -17
  796. package/src/daemon/message-types/subagents.ts +0 -63
  797. package/src/daemon/message-types/upgrades.ts +0 -14
  798. package/src/daemon/message-types/workflows.ts +0 -25
  799. package/src/daemon/message-types/workspace.ts +0 -137
  800. package/src/live-voice/ack-phrases.ts +0 -58
  801. package/src/plugins/defaults/memory/retrieval-budget.ts +0 -29
  802. package/src/plugins/defaults/memory/v3/__tests__/gate-flag.test.ts +0 -69
  803. package/src/plugins/defaults/memory/v3/gate-flag.ts +0 -20
  804. package/src/plugins/defaults/memory/v3/substrate/__tests__/memory-v2-startup.test.ts +0 -86
  805. package/src/plugins/defaults/memory/v3/substrate/memory-v2-startup.test.ts +0 -205
  806. /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/concept-page-frontmatter-schema.test.ts +0 -0
  807. /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-prompt-flag-gating-guard.test.ts +0 -0
  808. /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/injected-block-slugs.test.ts +0 -0
  809. /package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-content.ts +0 -0
@@ -6,7 +6,12 @@ import { ProviderError, type ProviderErrorReason } from "../../util/errors.js";
6
6
  import { getLogger } from "../../util/logger.js";
7
7
  import { extractRetryAfterMs } from "../../util/retry.js";
8
8
  import { escapeXmlAttr } from "../../util/xml.js";
9
- import { base64Source, resolveMediaReferences } from "../media-resolve.js";
9
+ import {
10
+ base64Source,
11
+ mediaSourceByteLength,
12
+ resolveMediaReferences,
13
+ } from "../media-resolve.js";
14
+ import { modelSupportsAudioInput } from "../model-catalog.js";
10
15
  import { PLACEHOLDER_EMPTY_TURN } from "../placeholder-sentinels.js";
11
16
  import { createStreamTimeout } from "../stream-timeout.js";
12
17
  import { createToolProgressEmitter } from "../tool-progress-events.js";
@@ -35,6 +40,11 @@ import {
35
40
  coerceObjectParamsToJsonString,
36
41
  decodeCoercedObjectArgs,
37
42
  } from "./coerce-object-args.js";
43
+ import {
44
+ isOpenAICompatInlineAudio,
45
+ OPENAI_COMPAT_MAX_INLINE_AUDIO_BYTES,
46
+ openAIInputAudioFormat,
47
+ } from "./input-audio.js";
38
48
 
39
49
  /**
40
50
  * Detect OpenAI-compatible context-overflow signals on an `OpenAI.APIError`.
@@ -336,6 +346,10 @@ export class OpenAIChatCompletionsProvider implements Provider {
336
346
  options.coerceObjectArgsToJsonString ?? false;
337
347
  }
338
348
 
349
+ get defaultModel(): string {
350
+ return this.model;
351
+ }
352
+
339
353
  async sendMessage(
340
354
  messages: Message[],
341
355
  options?: SendMessageOptions,
@@ -359,7 +373,11 @@ export class OpenAIChatCompletionsProvider implements Provider {
359
373
  const coercedObjectKeys = new Map<string, string[]>();
360
374
 
361
375
  try {
362
- const openaiMessages = this.toOpenAIMessages(messages, systemPrompt);
376
+ const openaiMessages = this.toOpenAIMessages(
377
+ messages,
378
+ systemPrompt,
379
+ modelSupportsAudioInput(modelOverride ?? this.model),
380
+ );
363
381
 
364
382
  const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming =
365
383
  {
@@ -893,6 +911,7 @@ export class OpenAIChatCompletionsProvider implements Provider {
893
911
  private toOpenAIMessages(
894
912
  messages: Message[],
895
913
  systemPrompt?: string,
914
+ audioInputEnabled = false,
896
915
  ): OpenAI.Chat.Completions.ChatCompletionMessageParam[] {
897
916
  // Swap any persisted attachment references back to inline base64 before
898
917
  // serializing, so the block transforms below can read `source.data`.
@@ -923,9 +942,9 @@ export class OpenAIChatCompletionsProvider implements Provider {
923
942
  );
924
943
 
925
944
  // Emit tool results as separate tool-role messages
926
- // OpenAI's API only supports string content in tool messages, so images
927
- // from contentBlocks are collected and injected as a user message below.
928
- const toolResultImages: ContentBlock[] = [];
945
+ // OpenAI's API only supports string content in tool messages, so media
946
+ // from contentBlocks is collected and injected as a user message below.
947
+ const toolResultMedia: ContentBlock[] = [];
929
948
  for (const tr of toolResults) {
930
949
  let textContent = tr.content;
931
950
  if (tr.contentBlocks && tr.contentBlocks.length > 0) {
@@ -939,7 +958,18 @@ export class OpenAIChatCompletionsProvider implements Provider {
939
958
  textContent = textContent + "\n" + extraText.join("\n");
940
959
  }
941
960
  for (const cb of tr.contentBlocks) {
942
- if (cb.type === "image") toolResultImages.push(cb);
961
+ if (cb.type === "image") {
962
+ toolResultMedia.push(cb);
963
+ } else if (
964
+ audioInputEnabled &&
965
+ cb.type === "file" &&
966
+ isOpenAICompatInlineAudio(
967
+ cb.source.media_type,
968
+ mediaSourceByteLength(cb.source),
969
+ )
970
+ ) {
971
+ toolResultMedia.push(cb);
972
+ }
943
973
  }
944
974
  }
945
975
  result.push({
@@ -949,12 +979,13 @@ export class OpenAIChatCompletionsProvider implements Provider {
949
979
  });
950
980
  }
951
981
 
952
- // Emit remaining content + any tool result images as a user message.
953
- // Images from tool results (e.g. browser_screenshot) must go in a user
954
- // message because OpenAI-compatible APIs don't support images in tool messages.
955
- const userContent = [...otherBlocks, ...toolResultImages];
982
+ // Emit remaining content + any tool result media as a user message.
983
+ // Media from tool results (e.g. browser_screenshot, audio a tool read)
984
+ // must go in a user message because OpenAI-compatible APIs don't
985
+ // support media parts in tool messages.
986
+ const userContent = [...otherBlocks, ...toolResultMedia];
956
987
  if (userContent.length > 0) {
957
- result.push(this.toOpenAIUserMessage(userContent));
988
+ result.push(this.toOpenAIUserMessage(userContent, audioInputEnabled));
958
989
  }
959
990
  }
960
991
  }
@@ -1035,9 +1066,10 @@ export class OpenAIChatCompletionsProvider implements Provider {
1035
1066
  return result;
1036
1067
  }
1037
1068
 
1038
- /** Convert user content blocks (text, image) to an OpenAI user message. */
1069
+ /** Convert user content blocks (text, image, audio) to an OpenAI user message. */
1039
1070
  private toOpenAIUserMessage(
1040
1071
  blocks: ContentBlock[],
1072
+ audioInputEnabled = false,
1041
1073
  ): OpenAI.Chat.Completions.ChatCompletionUserMessageParam {
1042
1074
  // If only a single text block, use plain string (simpler, fewer tokens)
1043
1075
  if (blocks.length === 1 && blocks[0].type === "text") {
@@ -1066,12 +1098,28 @@ export class OpenAIChatCompletionsProvider implements Provider {
1066
1098
  });
1067
1099
  }
1068
1100
  break;
1069
- case "file":
1070
- parts.push({
1071
- type: "text",
1072
- text: this.fileBlockToText(block),
1073
- });
1101
+ case "file": {
1102
+ const audioFormat = audioInputEnabled
1103
+ ? openAIInputAudioFormat(block.source.media_type)
1104
+ : null;
1105
+ if (
1106
+ audioFormat !== null &&
1107
+ mediaSourceByteLength(block.source) <=
1108
+ OPENAI_COMPAT_MAX_INLINE_AUDIO_BYTES
1109
+ ) {
1110
+ const audioSrc = base64Source(block.source);
1111
+ parts.push({
1112
+ type: "input_audio",
1113
+ input_audio: { data: audioSrc.data, format: audioFormat },
1114
+ });
1115
+ } else {
1116
+ parts.push({
1117
+ type: "text",
1118
+ text: this.fileBlockToText(block),
1119
+ });
1120
+ }
1074
1121
  break;
1122
+ }
1075
1123
  case "server_tool_use":
1076
1124
  parts.push({
1077
1125
  type: "text",
@@ -0,0 +1,76 @@
1
+ /**
2
+ * Shared source of truth for how OpenAI-compatible chat-completions providers
3
+ * ingest inline audio: which MIME types map onto the wire `input_audio.format`
4
+ * enum, the inline size cap, and the token-cost heuristic. The provider
5
+ * transform (`chat-completions-provider.ts`) and the context budgeter
6
+ * (`context/token-estimator.ts`) both read these — if they disagreed about
7
+ * which audio is sent inline, the budgeter would mis-count what the provider
8
+ * actually ships.
9
+ *
10
+ * Wire shape (OpenAI chat-completions spec):
11
+ * { type: "input_audio", input_audio: { data: <base64>, format: "wav"|"mp3" } }
12
+ *
13
+ * m4a/ogg/flac/opus are deliberately absent: the spec's `format` enum is
14
+ * wav|mp3 only. Whether a given model receives audio at all is gated by the
15
+ * catalog `supportsAudioInput` capability (`modelSupportsAudioInput`), not
16
+ * here — this module only decides eligibility of the payload itself.
17
+ */
18
+
19
+ export type OpenAIInputAudioFormat = "wav" | "mp3";
20
+
21
+ const MIME_TO_INPUT_AUDIO_FORMAT: ReadonlyMap<string, OpenAIInputAudioFormat> =
22
+ new Map([
23
+ ["audio/wav", "wav"],
24
+ ["audio/x-wav", "wav"],
25
+ ["audio/wave", "wav"],
26
+ ["audio/mpeg", "mp3"],
27
+ ["audio/mp3", "mp3"],
28
+ ]);
29
+
30
+ /**
31
+ * Map a stored attachment MIME type onto the `input_audio.format` enum, or
32
+ * `null` when the type is not sent inline.
33
+ */
34
+ export function openAIInputAudioFormat(
35
+ mimeType: string,
36
+ ): OpenAIInputAudioFormat | null {
37
+ return MIME_TO_INPUT_AUDIO_FORMAT.get(mimeType.toLowerCase()) ?? null;
38
+ }
39
+
40
+ /**
41
+ * Max raw bytes of audio sent inline. Base64 inflates the payload ~4/3, so
42
+ * 12 MB raw ≈ 16 MB encoded — headroom for the rest of the prompt under
43
+ * typical request-size limits (same rationale as the Gemini inline-audio cap).
44
+ */
45
+ export const OPENAI_COMPAT_MAX_INLINE_AUDIO_BYTES = 12 * 1024 * 1024;
46
+
47
+ /**
48
+ * True when an audio media source of this MIME type and byte length is
49
+ * eligible to be sent inline as an `input_audio` part (given a model whose
50
+ * catalog entry carries `supportsAudioInput`).
51
+ */
52
+ export function isOpenAICompatInlineAudio(
53
+ mimeType: string,
54
+ byteLength: number,
55
+ ): boolean {
56
+ return (
57
+ openAIInputAudioFormat(mimeType) !== null &&
58
+ byteLength <= OPENAI_COMPAT_MAX_INLINE_AUDIO_BYTES
59
+ );
60
+ }
61
+
62
+ /**
63
+ * Audio-native chat-completions models bill audio by duration, not payload
64
+ * size (measured on Baseten's Inkling: ~20 tokens/sec — 127 audio tokens for
65
+ * a 6.3 s clip). Duration is approximated from byte length at compressed-audio
66
+ * rates (~16 KB/s); for uncompressed PCM wavs this overestimates duration and
67
+ * therefore tokens — the safe direction for the compaction pre-check
68
+ * (compact early, never overflow).
69
+ */
70
+ const AUDIO_TOKENS_PER_SECOND = 20;
71
+ const APPROX_AUDIO_BYTES_PER_SECOND = 16_000;
72
+
73
+ export function estimateOpenAICompatAudioTokens(byteLength: number): number {
74
+ const approxSeconds = byteLength / APPROX_AUDIO_BYTES_PER_SECOND;
75
+ return Math.ceil(approxSeconds * AUDIO_TOKENS_PER_SECOND);
76
+ }
@@ -118,7 +118,6 @@ export class OpenRouterResponsesProvider extends OpenAIResponsesProvider {
118
118
 
119
119
  export class OpenRouterProvider extends OpenAIChatCompletionsProvider {
120
120
  private readonly openRouterApiKey: string;
121
- private readonly defaultModel: string;
122
121
  private readonly resolvedBaseURL: string;
123
122
  private readonly providerStreamTimeoutMs: number | undefined;
124
123
  private readonly useNativeWebSearch: boolean;
@@ -141,7 +140,6 @@ export class OpenRouterProvider extends OpenAIChatCompletionsProvider {
141
140
  backfillEmptyAssistantContent: true,
142
141
  });
143
142
  this.openRouterApiKey = apiKey;
144
- this.defaultModel = model;
145
143
  this.resolvedBaseURL = baseURL;
146
144
  this.providerStreamTimeoutMs = options.streamTimeoutMs;
147
145
  this.useNativeWebSearch = options.useNativeWebSearch ?? false;
@@ -51,6 +51,10 @@ export const PLATFORM_PROVIDER_META: Record<string, ManagedProviderMeta> = {
51
51
  managed: true,
52
52
  proxyPath: "/v1/runtime-proxy/together",
53
53
  },
54
+ poolside: {
55
+ name: "poolside",
56
+ managed: false,
57
+ },
54
58
  openrouter: {
55
59
  name: "openrouter",
56
60
  managed: false,
@@ -0,0 +1,42 @@
1
+ import {
2
+ type ApiKeyValidationResult,
3
+ validateOpenAICompatibleApiKey,
4
+ } from "../openai/api-key-validation.js";
5
+ import { OpenAIChatCompletionsProvider } from "../openai/chat-completions-provider.js";
6
+
7
+ export interface PoolsideProviderOptions {
8
+ baseURL?: string;
9
+ streamTimeoutMs?: number;
10
+ }
11
+
12
+ /** Poolside exposes a single OpenAI-compatible inference endpoint. */
13
+ const DEFAULT_POOLSIDE_BASE_URL = "https://inference.poolside.ai";
14
+
15
+ export async function validatePoolsideApiKey(
16
+ apiKey: string,
17
+ ): Promise<ApiKeyValidationResult> {
18
+ return validateOpenAICompatibleApiKey({
19
+ apiKey,
20
+ baseURL: DEFAULT_POOLSIDE_BASE_URL,
21
+ providerLabel: "Poolside",
22
+ });
23
+ }
24
+
25
+ export class PoolsideProvider extends OpenAIChatCompletionsProvider {
26
+ constructor(
27
+ apiKey: string,
28
+ model: string,
29
+ options: PoolsideProviderOptions = {},
30
+ ) {
31
+ super(apiKey, model, {
32
+ baseURL: options.baseURL?.trim() || DEFAULT_POOLSIDE_BASE_URL,
33
+ providerName: "poolside",
34
+ providerLabel: "Poolside",
35
+ streamTimeoutMs: options.streamTimeoutMs,
36
+ // Poolside's Laguna models emit chain of thought via `reasoning_content`
37
+ // rather than inline `<think>` tags. The base provider parses this field
38
+ // into thinking blocks and replays it on multi-turn requests.
39
+ assistantReasoningField: "reasoning_content",
40
+ });
41
+ }
42
+ }
@@ -35,8 +35,32 @@ const providers = new Map<string, Provider>();
35
35
  const routingSources = new Map<string, "user-key" | "managed-proxy">();
36
36
  const NATIVE_WEB_SEARCH_PROVIDER_IDS = new Set(["anthropic", "openai"]);
37
37
 
38
+ /**
39
+ * A cached per-connection provider and the wall-clock time it goes stale.
40
+ *
41
+ * The credential store is the source of truth for a connection's API key; a
42
+ * resolved provider bakes that key into its auth headers and is only a cache of
43
+ * it. `expiresAt` bounds how long a cached provider is trusted before the key
44
+ * is re-read from the store, so a credential rotated out-of-band (e.g. the
45
+ * platform reprovisioning the assistant API key) is picked up on its own — in
46
+ * every process that holds this cache, with no cross-process invalidation.
47
+ */
48
+ interface CachedConnectionProvider {
49
+ provider: Provider;
50
+ expiresAt: number;
51
+ }
52
+
53
+ /**
54
+ * How long a resolved per-connection provider is served from cache before it is
55
+ * re-resolved from the credential store. Bounds credential staleness in any
56
+ * process holding the cache (the daemon and each sidecar worker) to this
57
+ * window. Kept short enough that a rotated key self-heals quickly, long enough
58
+ * that repeated resolutions within a burst still reuse one adapter.
59
+ */
60
+ const CONNECTION_PROVIDER_CACHE_TTL_MS = 60_000;
61
+
38
62
  /** Per-connection provider cache, keyed by connection name, effective provider, and model. */
39
- const connectionProviders = new Map<string, Provider>();
63
+ const connectionProviders = new Map<string, CachedConnectionProvider>();
40
64
 
41
65
  function getConnectionProviderCacheKey(
42
66
  connection: ProviderConnection,
@@ -75,7 +99,6 @@ export interface ProvidersConfig {
75
99
  services: {
76
100
  inference: Record<string, never>;
77
101
  "image-generation": {
78
- mode: "managed" | "your-own";
79
102
  provider: string;
80
103
  model: string;
81
104
  };
@@ -257,9 +280,12 @@ export async function initializeProviders(
257
280
  /**
258
281
  * Resolve a provider instance for a named `provider_connection`.
259
282
  *
260
- * Results are cached in `connectionProviders` for the lifetime of the
261
- * current `initializeProviders` invocation (cleared on next boot). This
262
- * prevents redundant vault reads for repeated calls to the same connection.
283
+ * Results are cached in `connectionProviders` to avoid redundant vault reads
284
+ * for repeated calls to the same connection. A cached provider bakes in the
285
+ * credential it resolved, so each entry carries a TTL
286
+ * ({@link CONNECTION_PROVIDER_CACHE_TTL_MS}): once it expires the credential is
287
+ * re-read from the store, bounding how long an out-of-band key rotation stays
288
+ * stale. Entries are also cleared eagerly on connection mutations and boot.
263
289
  *
264
290
  * Returns null when:
265
291
  * - The connection doesn't exist in the DB
@@ -292,9 +318,12 @@ export async function resolveProviderFromConnection(
292
318
  effectiveProvider,
293
319
  );
294
320
  const cached = connectionProviders.get(cacheKey);
295
- if (cached) {
296
- return cached;
321
+ if (cached && cached.expiresAt > Date.now()) {
322
+ return cached.provider;
297
323
  }
324
+ // A stale entry is a miss: fall through to re-resolve. The credential is
325
+ // re-read from the store below, so a key rotated since the entry was cached
326
+ // is picked up here rather than served stale forever.
298
327
 
299
328
  const authResult = await resolveAuth(connection.auth, effectiveProvider, {
300
329
  baseUrl: connection.baseUrl,
@@ -341,7 +370,10 @@ export async function resolveProviderFromConnection(
341
370
  );
342
371
 
343
372
  if (provider) {
344
- connectionProviders.set(cacheKey, provider);
373
+ connectionProviders.set(cacheKey, {
374
+ provider,
375
+ expiresAt: Date.now() + CONNECTION_PROVIDER_CACHE_TTL_MS,
376
+ });
345
377
  }
346
378
 
347
379
  return provider;
@@ -57,6 +57,7 @@ const EFFORT_SUPPORTED_PROVIDERS = new Set([
57
57
  "fireworks",
58
58
  "together",
59
59
  "baseten",
60
+ "poolside",
60
61
  ]);
61
62
 
62
63
  // For these providers, disabling reasoning is encoded through the same effort
@@ -69,6 +70,7 @@ const DISABLED_THINKING_USES_EFFORT_PROVIDERS = new Set([
69
70
  "openrouter",
70
71
  "vercel-ai-gateway",
71
72
  "baseten",
73
+ "poolside",
72
74
  ]);
73
75
 
74
76
  // Whether a disabled `thinking` config must be encoded as `effort: "none"`
@@ -128,6 +128,36 @@ export interface WebSearchToolResultContent {
128
128
  content: unknown; // Opaque — encrypted_content in search results is provider-specific
129
129
  }
130
130
 
131
+ /**
132
+ * A client-rendered UI card persisted into conversation history: call
133
+ * summaries, guardian approval cards, skill cards, wake notices, document
134
+ * previews.
135
+ *
136
+ * This is a rendering instruction, not model context — providers drop it when
137
+ * serializing history. Producers therefore pair the surface with a sibling
138
+ * `text` block flagged `_surfaceFallback` (see
139
+ * `notifications/approval-card-builder.ts`); that text is what feeds the model,
140
+ * search indexing, CLI display, and channel replies.
141
+ *
142
+ * `data` is deliberately opaque: its concrete shape is selected by
143
+ * `surfaceType` and owned by `daemon/message-types/surfaces.ts`.
144
+ */
145
+ export interface UiSurfaceContent {
146
+ type: "ui_surface";
147
+ surfaceId: string;
148
+ surfaceType: string;
149
+ title?: string;
150
+ data?: Record<string, unknown>;
151
+ actions?: unknown[];
152
+ /**
153
+ * Free-form, matching `CurrentTurnSurface.display` — NOT the `inline` /
154
+ * `panel` enum of the `ui_surface_show` wire event. Persisted surfaces carry
155
+ * whatever the `ui_show` tool wrote, so this must not narrow.
156
+ */
157
+ display?: string;
158
+ completed?: boolean;
159
+ }
160
+
131
161
  export type ContentBlock =
132
162
  | TextContent
133
163
  | ThinkingContent
@@ -137,7 +167,8 @@ export type ContentBlock =
137
167
  | ToolUseContent
138
168
  | ToolResultContent
139
169
  | ServerToolUseContent
140
- | WebSearchToolResultContent;
170
+ | WebSearchToolResultContent
171
+ | UiSurfaceContent;
141
172
 
142
173
  export interface Message {
143
174
  role: "user" | "assistant";
@@ -337,6 +368,13 @@ export interface Provider {
337
368
  * Falls back to `name` when unset.
338
369
  */
339
370
  tokenEstimationProvider?: string;
371
+ /**
372
+ * Model id this instance dispatches when a call carries no per-call model
373
+ * override. Consumed by the local token estimator for model-keyed rules
374
+ * (e.g. audio-capable OpenAI-compatible models). Optional: providers whose
375
+ * estimation rules are provider-wide need not expose it.
376
+ */
377
+ defaultModel?: string;
340
378
  /**
341
379
  * True when this provider instance was constructed to run web search
342
380
  * server-side (provider-native). The native search only activates when a
@@ -26,7 +26,6 @@ const DEFAULT_VERCEL_AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1";
26
26
 
27
27
  export class VercelAIGatewayProvider extends OpenAIChatCompletionsProvider {
28
28
  private readonly gatewayApiKey: string;
29
- private readonly defaultModel: string;
30
29
  private readonly resolvedBaseURL: string;
31
30
  private readonly providerStreamTimeoutMs: number | undefined;
32
31
  private readonly useNativeWebSearch: boolean;
@@ -50,7 +49,6 @@ export class VercelAIGatewayProvider extends OpenAIChatCompletionsProvider {
50
49
  backfillEmptyAssistantContent: true,
51
50
  });
52
51
  this.gatewayApiKey = apiKey;
53
- this.defaultModel = model;
54
52
  this.resolvedBaseURL = baseURL;
55
53
  this.providerStreamTimeoutMs = options.streamTimeoutMs;
56
54
  this.useNativeWebSearch = options.useNativeWebSearch ?? false;
@@ -157,13 +157,15 @@ All CDP-backed browser tools (`browser_navigate`, `browser_snapshot`, `browser_s
157
157
 
158
158
  **Test coverage:** Regression tests for `browser_mode` wiring live in `__tests__/headless-browser-mode.test.ts`. E2E regression tests for backend precedence live in `__tests__/host-browser-e2e-cloud.test.ts` (extension path and macOS SSE bridge path) and `__tests__/conversation-routes-disk-view.test.ts` (macOS fallback path). Unit tests for pinned candidate construction and failover live in `tools/browser/cdp-client/__tests__/factory.test.ts`. Browser status tests covering macOS host-browser diagnostics live in `tools/browser/__tests__/browser-status.test.ts`.
159
159
 
160
- ### Channel approvals (Telegram, Slack)
160
+ ### Interactive requests on channels (approvals, questions)
161
161
 
162
- Channel approval flows use `requestId` (not `runId`) as the primary identifier:
162
+ **The guardian-request pipeline is the canonical rail for anything interactive on a channel** — cards with buttons, request-code replies, emoji reactions, typed answers. The end-to-end map (promotion → gateway `guardian_requests` row → notification broadcaster → per-channel adapters → reply router → decision primitive → per-kind resolver) lives in [docs/guardian-request-flow.md](../../docs/guardian-request-flow.md). New interactive features extend that pipeline's seams; do NOT add per-feature watchers, callback schemes, or inbound intercepts.
163
163
 
164
- - Telegram callback buttons encode `apr:<requestId>:<action>` in `callback_data`.
165
- - Guardian approval records in the gateway-owned `guardian_requests` table (and their `guardian_request_deliveries`) link via `requestId`; the daemon reads/writes them through the `channels/gateway-guardian-requests.ts` client.
166
- - The conversational approval engine classifies user intent and resolves via `conversation.handleConfirmationResponse(requestId, decision)`.
164
+ Identifiers and plumbing notes:
165
+
166
+ - Channel flows use `requestId` (not `runId`) as the primary identifier. Callback buttons encode `apr:<requestId>:<action>` in `callback_data`; answer-option buttons on question cards use action tokens `answer_<idx>` / `answer_skip` under the same prefix.
167
+ - Guardian request records live in the gateway-owned `guardian_requests` table (and their `guardian_request_deliveries`); the daemon reads/writes them through the `channels/gateway-guardian-requests.ts` client.
168
+ - Legacy in-turn interception (`routes/guardian-approval-interception.ts` + the approval prompt watcher) still serves a guardian's own tool-approval prompts mid-turn, resolving via `conversation.handleConfirmationResponse(requestId, decision)`. It is a legacy rail — the reply router runs first; converge new work on the pipeline.
167
169
 
168
170
  ### Channel verification: gateway-owned
169
171
 
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Tests for the question → guardian.question bridge
3
+ * (`question-request-guardian-bridge.ts`): the emitted signal must be
4
+ * deliverable (NOT source-active suppressed) and carry the option payload the
5
+ * broadcaster renders; identity guards skip emission rather than misroute.
6
+ */
7
+
8
+ import { beforeEach, describe, expect, mock, test } from "bun:test";
9
+
10
+ import type { GuardianRequestWire } from "../../channels/gateway-guardian-requests.js";
11
+
12
+ const emitSignalMock = mock(
13
+ (_params: Record<string, unknown>): Promise<Record<string, unknown>> =>
14
+ Promise.resolve({ signalId: "sig-1", deliveryResults: [] }),
15
+ );
16
+ const recordCardDeliveryMock = mock((_params: Record<string, unknown>) =>
17
+ Promise.resolve(undefined),
18
+ );
19
+ const recordDeliveriesMock = mock((_params: Record<string, unknown>) =>
20
+ Promise.resolve(),
21
+ );
22
+
23
+ let binding: { guardianExternalUserId: string } | null = null;
24
+
25
+ mock.module("../../notifications/emit-signal.js", () => ({
26
+ emitNotificationSignal: (params: Record<string, unknown>) =>
27
+ emitSignalMock(params),
28
+ }));
29
+ mock.module("../../notifications/guardian-delivery-recorder.js", () => ({
30
+ recordApprovalCardDelivery: (params: Record<string, unknown>) =>
31
+ recordCardDeliveryMock(params),
32
+ recordGuardianRequestDeliveries: (params: Record<string, unknown>) =>
33
+ recordDeliveriesMock(params),
34
+ }));
35
+ mock.module("../channel-verification-service.js", () => ({
36
+ getGuardianBinding: () => Promise.resolve(binding),
37
+ }));
38
+ // The vellum-card pin mirrors the target conversation's source; no DB here,
39
+ // so resolve "conversation not found" and let the helper fall back to "user".
40
+ mock.module("../../persistence/conversation-crud.js", () => ({
41
+ getConversation: () => null,
42
+ }));
43
+
44
+ const { bridgeQuestionRequestToGuardian } =
45
+ await import("../question-request-guardian-bridge.js");
46
+
47
+ function makeRequest(
48
+ overrides: Partial<GuardianRequestWire> = {},
49
+ ): GuardianRequestWire {
50
+ return {
51
+ id: "req-1",
52
+ kind: "pending_question",
53
+ sourceType: "channel",
54
+ sourceChannel: "telegram",
55
+ sourceConversationId: "conv-1",
56
+ requesterExternalUserId: "tg-guardian",
57
+ requesterChatId: "chat-1",
58
+ guardianExternalUserId: "tg-guardian",
59
+ guardianPrincipalId: "prin-guardian",
60
+ callSessionId: null,
61
+ pendingQuestionId: null,
62
+ questionText: "Which fruit?",
63
+ requestCode: "ABC123",
64
+ toolName: null,
65
+ inputDigest: null,
66
+ commandPreview: null,
67
+ riskLevel: null,
68
+ activityText: null,
69
+ executionTarget: null,
70
+ requesterSignals: null,
71
+ requestTrigger: null,
72
+ status: "pending",
73
+ answerText: null,
74
+ decidedByExternalUserId: null,
75
+ decidedByPrincipalId: null,
76
+ followupState: null,
77
+ expiresAt: null,
78
+ createdAt: 0,
79
+ updatedAt: 0,
80
+ ...overrides,
81
+ };
82
+ }
83
+
84
+ const trustContext = {
85
+ trustClass: "guardian",
86
+ sourceChannel: "telegram",
87
+ guardianPrincipalId: "prin-guardian",
88
+ guardianExternalUserId: "tg-guardian",
89
+ requesterExternalUserId: "tg-guardian",
90
+ requesterChatId: "chat-1",
91
+ } as never;
92
+
93
+ const question = {
94
+ id: "q1",
95
+ question: "Which fruit?",
96
+ options: [
97
+ { id: "apple", label: "Apple" },
98
+ { id: "banana", label: "Banana" },
99
+ ],
100
+ };
101
+
102
+ beforeEach(() => {
103
+ emitSignalMock.mockClear();
104
+ recordCardDeliveryMock.mockClear();
105
+ recordDeliveriesMock.mockClear();
106
+ binding = { guardianExternalUserId: "tg-guardian" };
107
+ });
108
+
109
+ describe("bridgeQuestionRequestToGuardian", () => {
110
+ test("emits a DELIVERABLE guardian.question signal carrying the options", async () => {
111
+ const result = await bridgeQuestionRequestToGuardian({
112
+ guardianRequest: makeRequest(),
113
+ trustContext,
114
+ conversationId: "conv-1",
115
+ question,
116
+ });
117
+
118
+ expect(result).toMatchObject({ bridged: true });
119
+ expect(emitSignalMock).toHaveBeenCalledTimes(1);
120
+ const signal = emitSignalMock.mock.calls[0]?.[0] as {
121
+ attentionHints: Record<string, unknown>;
122
+ contextPayload: Record<string, unknown>;
123
+ conversationAffinityHint?: Record<string, string>;
124
+ };
125
+ // The in-app card is pinned to the conversation the question was asked
126
+ // in — never left to LLM conversation routing.
127
+ expect(signal.conversationAffinityHint).toEqual({ vellum: "conv-1" });
128
+ // REGRESSION PIN: visibleInSourceNow is a hard suppression pre-gate in
129
+ // emitNotificationSignal. The question card is the channel's ONLY way to
130
+ // display the parked prompt — a true here silently suppresses every
131
+ // question card and the tool hangs until the prompt timeout.
132
+ expect(signal.attentionHints.visibleInSourceNow).toBe(false);
133
+ expect(signal.contextPayload).toMatchObject({
134
+ requestKind: "pending_question",
135
+ requestId: "req-1",
136
+ questionText: "Which fruit?",
137
+ options: [
138
+ { id: "apple", label: "Apple" },
139
+ { id: "banana", label: "Banana" },
140
+ ],
141
+ });
142
+ });
143
+
144
+ test("skips emission when no guardian binding exists for the channel", async () => {
145
+ binding = null;
146
+ const result = await bridgeQuestionRequestToGuardian({
147
+ guardianRequest: makeRequest(),
148
+ trustContext,
149
+ conversationId: "conv-1",
150
+ question,
151
+ });
152
+ expect(result).toMatchObject({
153
+ skipped: true,
154
+ reason: "no_guardian_binding",
155
+ });
156
+ expect(emitSignalMock).not.toHaveBeenCalled();
157
+ });
158
+
159
+ test("skips emission when the binding identity no longer matches (rebind guard)", async () => {
160
+ binding = { guardianExternalUserId: "tg-someone-else" };
161
+ const result = await bridgeQuestionRequestToGuardian({
162
+ guardianRequest: makeRequest(),
163
+ trustContext,
164
+ conversationId: "conv-1",
165
+ question,
166
+ });
167
+ expect(result).toMatchObject({
168
+ skipped: true,
169
+ reason: "binding_identity_mismatch",
170
+ });
171
+ expect(emitSignalMock).not.toHaveBeenCalled();
172
+ });
173
+ });