@vellumai/assistant 0.12.2 → 0.12.3-dev.202609220224.aacf886

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (792) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/README.md +12 -12
  6. package/docs/architecture/integrations.md +7 -0
  7. package/docs/architecture/memory.md +17 -2
  8. package/docs/architecture/security.md +10 -5
  9. package/docs/architecture/turn-actor.md +2 -0
  10. package/docs/credential-execution-service.md +1 -1
  11. package/docs/desktop-browser-cli.md +50 -5
  12. package/docs/live-voice-task-outcomes.md +100 -0
  13. package/docs/mode-sessions.md +183 -0
  14. package/docs/voice-input-diagnostics.md +58 -0
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  19. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  20. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  21. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  26. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  27. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  28. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  29. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  30. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  31. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  32. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  33. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  35. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  36. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  37. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  38. package/openapi.yaml +716 -16
  39. package/package.json +1 -1
  40. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  41. package/scripts/smoke-desktop-window-theme.ts +2 -3
  42. package/scripts/sync-llm-catalog.ts +3 -0
  43. package/scripts/voice-ttft-spike.ts +2 -2
  44. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  45. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  46. package/src/__tests__/agent-loop.test.ts +259 -0
  47. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  48. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  49. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  50. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  51. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  52. package/src/__tests__/anthropic-provider.test.ts +55 -0
  53. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  54. package/src/__tests__/approval-message-composer.test.ts +1 -118
  55. package/src/__tests__/approval-routes-http.test.ts +7 -2
  56. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  57. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  58. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  59. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  60. package/src/__tests__/attachments-store.test.ts +22 -3
  61. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  62. package/src/__tests__/btw-routes.test.ts +4 -1
  63. package/src/__tests__/build-persisted-content.test.ts +1 -1
  64. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  65. package/src/__tests__/call-recovery.test.ts +7 -2
  66. package/src/__tests__/call-routes-http.test.ts +16 -2
  67. package/src/__tests__/call-store.test.ts +7 -2
  68. package/src/__tests__/channel-approval.test.ts +9 -14
  69. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  70. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  71. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  72. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  73. package/src/__tests__/checker.test.ts +11 -3
  74. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  75. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  76. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  77. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  78. package/src/__tests__/computer-use-tools.test.ts +1 -233
  79. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  80. package/src/__tests__/config-schema.test.ts +16 -0
  81. package/src/__tests__/config-watcher.test.ts +24 -0
  82. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  83. package/src/__tests__/context-search-fanout.test.ts +1 -1
  84. package/src/__tests__/context-search-types.test.ts +7 -7
  85. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  86. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  89. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  90. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  91. package/src/__tests__/conversation-attachments.test.ts +106 -0
  92. package/src/__tests__/conversation-evictor.test.ts +62 -37
  93. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  94. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  95. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  96. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  97. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  98. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  99. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  101. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  102. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  103. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  104. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  105. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  106. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  107. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  108. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  109. package/src/__tests__/conversation-undo.test.ts +61 -0
  110. package/src/__tests__/credential-execution-client.test.ts +72 -55
  111. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  112. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  113. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  114. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  115. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  116. package/src/__tests__/dm-backfill.test.ts +91 -1
  117. package/src/__tests__/document-create-route.test.ts +88 -0
  118. package/src/__tests__/document-sync-tags.test.ts +27 -0
  119. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  120. package/src/__tests__/events-tail-route.test.ts +58 -0
  121. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  122. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  123. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  124. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  125. package/src/__tests__/heartbeat-service.test.ts +2 -59
  126. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  127. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  128. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  129. package/src/__tests__/history-repair.test.ts +53 -0
  130. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  131. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  132. package/src/__tests__/image-credentials.test.ts +49 -0
  133. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  134. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  135. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  136. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  137. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  138. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  139. package/src/__tests__/media-generate-image.test.ts +49 -1
  140. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  141. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  142. package/src/__tests__/mode-session-history.test.ts +245 -0
  143. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  144. package/src/__tests__/mode-session-wire.test.ts +141 -0
  145. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  146. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  147. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  148. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  149. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  150. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  151. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  152. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  153. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  154. package/src/__tests__/persist-media-references.test.ts +50 -0
  155. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  156. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  157. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  158. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  159. package/src/__tests__/process-message-display-content.test.ts +56 -0
  160. package/src/__tests__/provider-platform-proxy-integration.test.ts +45 -15
  161. package/src/__tests__/reaction-persistence.test.ts +5 -0
  162. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  163. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  164. package/src/__tests__/runtime-attachment-metadata.test.ts +4 -2
  165. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  166. package/src/__tests__/schedule-routes.test.ts +20 -0
  167. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  168. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  169. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  170. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  171. package/src/__tests__/secret-routes-platform-proxy.test.ts +87 -31
  172. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  173. package/src/__tests__/secret-scanner.test.ts +20 -0
  174. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  175. package/src/__tests__/secure-keys.test.ts +50 -4
  176. package/src/__tests__/send-endpoint-busy.test.ts +167 -3
  177. package/src/__tests__/server-history-render.test.ts +91 -1
  178. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  179. package/src/__tests__/settings-routes.test.ts +50 -1
  180. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  181. package/src/__tests__/skills.test.ts +5 -4
  182. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  183. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  184. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  185. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  186. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  187. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  188. package/src/__tests__/terminal-tools.test.ts +8 -0
  189. package/src/__tests__/thread-backfill.test.ts +53 -0
  190. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  191. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  192. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  193. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  194. package/src/__tests__/twilio-routes.test.ts +1 -0
  195. package/src/__tests__/unicode.test.ts +36 -0
  196. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  197. package/src/activation/progress-store.test.ts +12 -7
  198. package/src/agent/loop.ts +136 -53
  199. package/src/agent/tool-result-follow-up.ts +70 -0
  200. package/src/api/attachment-provenance.test.ts +68 -0
  201. package/src/api/computer-use-tool.test.ts +55 -0
  202. package/src/api/computer-use-tool.ts +27 -0
  203. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  204. package/src/api/events/assistant-turn-start.ts +3 -0
  205. package/src/api/events/desktop-activity-changed.ts +10 -0
  206. package/src/api/events/generation-handoff.ts +2 -0
  207. package/src/api/events/message-complete.ts +2 -0
  208. package/src/api/events/notification-conversation-created.ts +14 -16
  209. package/src/api/events/notification-intent.ts +4 -2
  210. package/src/api/events/question-request.ts +1 -0
  211. package/src/api/events/tool-result.ts +85 -0
  212. package/src/api/events/tool-use-start.ts +3 -0
  213. package/src/api/events/user-message-echo.ts +3 -0
  214. package/src/api/index.ts +42 -6
  215. package/src/api/mode-session.test.ts +131 -0
  216. package/src/api/mode-session.ts +148 -0
  217. package/src/api/remember-tool.ts +15 -0
  218. package/src/api/responses/conversation-message.ts +9 -0
  219. package/src/approvals/AGENTS.md +4 -2
  220. package/src/approvals/approval-primitive.ts +5 -2
  221. package/src/approvals/scoped-approval-grants.ts +6 -2
  222. package/src/browser/__tests__/operations.test.ts +29 -0
  223. package/src/browser/operation-meta.ts +2 -1
  224. package/src/browser/operations.ts +29 -0
  225. package/src/browser/virtual-desktop-target.ts +2 -9
  226. package/src/bundler/bundle-signer.ts +1 -3
  227. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  228. package/src/calls/__tests__/call-controller.test.ts +94 -0
  229. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  230. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  231. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  232. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  233. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  234. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  235. package/src/calls/barge-in-guard.ts +5 -0
  236. package/src/calls/call-controller.ts +196 -1
  237. package/src/calls/call-domain.ts +4 -0
  238. package/src/calls/call-store.ts +87 -2
  239. package/src/calls/guardian-action-sweep.ts +22 -8
  240. package/src/calls/media-stream-audio-transcode.ts +1 -3
  241. package/src/calls/typesafe-noul.ts +156 -0
  242. package/src/calls/voice-control-protocol.ts +6 -0
  243. package/src/calls/voice-escalation-judge.ts +145 -0
  244. package/src/calls/voice-escalation-target.ts +10 -0
  245. package/src/calls/voice-leg-coordinator.ts +121 -3
  246. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  247. package/src/calls/voice-session-bridge.ts +256 -112
  248. package/src/calls/voice-triage-escalate.ts +7 -0
  249. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  250. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  251. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  252. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  253. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  254. package/src/cli/commands/bash.help.ts +4 -3
  255. package/src/cli/commands/browser.help.ts +9 -6
  256. package/src/cli/commands/browser.ts +0 -1
  257. package/src/cli/commands/channel-verification-sessions.help.ts +12 -2
  258. package/src/cli/commands/clients.help.ts +46 -0
  259. package/src/cli/commands/clients.ts +116 -0
  260. package/src/cli/commands/credentials.help.ts +9 -4
  261. package/src/cli/commands/notifications.help.ts +16 -2
  262. package/src/cli/commands/notifications.ts +30 -8
  263. package/src/cli/commands/oauth/index.help.ts +10 -0
  264. package/src/cli/commands/oauth/providers.ts +12 -0
  265. package/src/cli/commands/oauth/request.test.ts +179 -2
  266. package/src/cli/commands/oauth/request.ts +41 -10
  267. package/src/cli/commands/plugins.ts +12 -5
  268. package/src/cli/commands/roadmap.help.ts +18 -21
  269. package/src/cli/commands/roadmap.ts +4 -21
  270. package/src/cli/commands/schedules.ts +2 -0
  271. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  272. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  273. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  274. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  275. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  276. package/src/cli/lib/bundled-marketplace.json +906 -1
  277. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  278. package/src/cli/lib/inspect-plugin.ts +11 -4
  279. package/src/cli/lib/plugin-marketplace.ts +8 -0
  280. package/src/cli/lib/publish-plugin.ts +1 -2
  281. package/src/cli/lib/upgrade-plugin.ts +19 -11
  282. package/src/cli/output.ts +18 -4
  283. package/src/config/__tests__/default-provider.test.ts +6 -0
  284. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  285. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  286. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  287. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  288. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  289. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  290. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  291. package/src/config/call-site-defaults.ts +9 -0
  292. package/src/config/feature-flag-registry.json +16 -8
  293. package/src/config/loader.ts +9 -17
  294. package/src/config/platform-identity.ts +8 -8
  295. package/src/config/profile-text-generation.test.ts +51 -0
  296. package/src/config/profile-text-generation.ts +51 -0
  297. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  298. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  299. package/src/config/schemas/call-site-catalog.ts +14 -0
  300. package/src/config/schemas/channels.ts +0 -18
  301. package/src/config/schemas/llm.ts +9 -2
  302. package/src/config/schemas/mcp.ts +6 -1
  303. package/src/config/schemas/memory-retrospective.ts +11 -2
  304. package/src/config/schemas/memory-v3.ts +12 -0
  305. package/src/config/schemas/services.ts +15 -6
  306. package/src/config/schemas/tools.ts +10 -0
  307. package/src/config/session-groups-gate.ts +21 -0
  308. package/src/config/skills.ts +2 -6
  309. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  310. package/src/conversations/message-consolidation.ts +102 -5
  311. package/src/credential-execution/ces-connect.test.ts +69 -0
  312. package/src/credential-execution/ces-connect.ts +115 -0
  313. package/src/credential-execution/ces-runtime.ts +55 -93
  314. package/src/credential-execution/executable-discovery.ts +34 -142
  315. package/src/credential-execution/process-manager.test.ts +20 -24
  316. package/src/credential-execution/process-manager.ts +24 -19
  317. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  318. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  319. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  320. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  321. package/src/daemon/assistant-attachments.ts +20 -12
  322. package/src/daemon/browser-mode-session.test.ts +402 -0
  323. package/src/daemon/browser-mode-session.ts +186 -0
  324. package/src/daemon/chat-credential-redaction.ts +16 -1
  325. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  326. package/src/daemon/computer-use-mode-session.ts +105 -0
  327. package/src/daemon/config-watcher.ts +10 -0
  328. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  329. package/src/daemon/conversation-agent-loop.ts +231 -25
  330. package/src/daemon/conversation-attachments.ts +93 -12
  331. package/src/daemon/conversation-evictor.ts +5 -24
  332. package/src/daemon/conversation-history.ts +10 -0
  333. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  334. package/src/daemon/conversation-interrupt.ts +7 -11
  335. package/src/daemon/conversation-messaging.ts +58 -5
  336. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  337. package/src/daemon/conversation-mode-session.ts +1008 -0
  338. package/src/daemon/conversation-process.ts +5 -0
  339. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  340. package/src/daemon/conversation-skill-tools.ts +1 -0
  341. package/src/daemon/conversation-store.ts +4 -4
  342. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  343. package/src/daemon/conversation-surfaces.ts +113 -5
  344. package/src/daemon/conversation-tool-setup.ts +79 -6
  345. package/src/daemon/conversation-turn-finalize.ts +26 -13
  346. package/src/daemon/conversation.ts +215 -58
  347. package/src/daemon/daemon-control.ts +3 -7
  348. package/src/daemon/handlers/config-channels.ts +3 -1
  349. package/src/daemon/handlers/config-model.ts +7 -5
  350. package/src/daemon/handlers/shared.ts +55 -16
  351. package/src/daemon/host-cu-proxy.ts +51 -22
  352. package/src/daemon/lifecycle.ts +30 -13
  353. package/src/daemon/mcp-reload-service.ts +1 -1
  354. package/src/daemon/message-protocol.ts +0 -1
  355. package/src/daemon/message-provenance.ts +28 -4
  356. package/src/daemon/message-types/shared.ts +1 -0
  357. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  358. package/src/daemon/mode-session-tracking.ts +54 -0
  359. package/src/daemon/orphan-reaper.ts +4 -3
  360. package/src/daemon/persist-media-references.ts +20 -3
  361. package/src/daemon/process-message.ts +13 -4
  362. package/src/daemon/tool-setup-types.ts +6 -0
  363. package/src/daemon/virtual-desktop-context.ts +21 -0
  364. package/src/daemon/wake-conversation-ops.ts +50 -15
  365. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  366. package/src/desktop/desktop-accessibility-script.ts +88 -0
  367. package/src/desktop/desktop-accessibility.test.ts +160 -0
  368. package/src/desktop/desktop-accessibility.ts +169 -0
  369. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  370. package/src/desktop/desktop-automation-lease.ts +129 -54
  371. package/src/desktop/desktop-browser-endpoint.ts +5 -0
  372. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  373. package/src/desktop/desktop-computer-use.test.ts +449 -0
  374. package/src/desktop/desktop-computer-use.ts +502 -0
  375. package/src/desktop/desktop-dependencies.ts +47 -293
  376. package/src/desktop/desktop-display.ts +2 -2
  377. package/src/desktop/desktop-help.ts +15 -0
  378. package/src/desktop/desktop-panel-config.test.ts +13 -3
  379. package/src/desktop/desktop-panel-config.ts +109 -15
  380. package/src/desktop/desktop-session-manager.test.ts +69 -21
  381. package/src/desktop/desktop-session-manager.ts +76 -22
  382. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  383. package/src/desktop/desktop-stream-bridge.ts +9 -11
  384. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  385. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  386. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  387. package/src/desktop/desktop-wallpaper.ts +34 -102
  388. package/src/desktop/desktop-window-theme.ts +5 -25
  389. package/src/desktop/virtual-desktop-feature.ts +53 -3
  390. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  391. package/src/documents/document-store.ts +23 -0
  392. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  393. package/src/heartbeat/heartbeat-service.ts +5 -33
  394. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  395. package/src/ipc/assistant-server.ts +2 -0
  396. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  397. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  398. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  399. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  400. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  401. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  402. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  403. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  404. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  405. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  406. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  407. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  408. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  409. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  410. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  411. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  412. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  413. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  414. package/src/live-voice/camera-mode-session.test.ts +265 -0
  415. package/src/live-voice/camera-mode-session.ts +277 -0
  416. package/src/live-voice/continuation-judge.ts +123 -0
  417. package/src/live-voice/live-activity-reporter.ts +115 -6
  418. package/src/live-voice/live-voice-manager.ts +36 -1
  419. package/src/live-voice/live-voice-photo.ts +88 -7
  420. package/src/live-voice/live-voice-session-manager.ts +15 -0
  421. package/src/live-voice/live-voice-session.ts +1443 -747
  422. package/src/live-voice/live-voice-tts.ts +22 -4
  423. package/src/live-voice/microphone-idle-gate.ts +56 -0
  424. package/src/live-voice/protocol.ts +171 -4
  425. package/src/live-voice/session-controls.ts +11 -3
  426. package/src/live-voice/task-outcomes.ts +131 -0
  427. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  428. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  429. package/src/mcp/__tests__/startup.test.ts +30 -7
  430. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  431. package/src/mcp/manager.ts +171 -101
  432. package/src/mcp/startup.ts +33 -13
  433. package/src/mcp/tool-caps.ts +184 -0
  434. package/src/media/__tests__/image-models.test.ts +58 -0
  435. package/src/media/image-credentials.ts +5 -0
  436. package/src/media/image-models.ts +52 -0
  437. package/src/media/image-service.ts +8 -0
  438. package/src/media/openrouter-image-service.test.ts +133 -0
  439. package/src/media/openrouter-image-service.ts +153 -0
  440. package/src/media/types.ts +1 -1
  441. package/src/messaging/provider-message-metadata.ts +3 -3
  442. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  443. package/src/messaging/providers/discord/send.ts +4 -17
  444. package/src/messaging/providers/discord/transport.ts +2 -2
  445. package/src/messaging/providers/outlook/client.ts +16 -22
  446. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  447. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  448. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  449. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  450. package/src/monitoring/__tests__/control.test.ts +17 -5
  451. package/src/monitoring/control.ts +15 -5
  452. package/src/notifications/AGENTS.md +1 -1
  453. package/src/notifications/README.md +83 -99
  454. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  455. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  456. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  457. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  458. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  459. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  460. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  461. package/src/notifications/adapters/macos.ts +37 -46
  462. package/src/notifications/broadcaster.ts +5 -6
  463. package/src/notifications/channel-allowlist.ts +44 -0
  464. package/src/notifications/copy-composer.ts +11 -26
  465. package/src/notifications/decision-engine.ts +24 -14
  466. package/src/notifications/deterministic-checks.ts +1 -1
  467. package/src/notifications/emit-signal.ts +123 -37
  468. package/src/notifications/home-feed-side-effect.ts +9 -7
  469. package/src/notifications/notification-utils.ts +19 -0
  470. package/src/notifications/schedule-result-producer.ts +4 -6
  471. package/src/notifications/signal.ts +4 -19
  472. package/src/notifications/types.ts +15 -0
  473. package/src/oauth/AGENTS.md +2 -0
  474. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  475. package/src/oauth/connection-resolver.ts +1 -2
  476. package/src/oauth/identity-verifier.ts +24 -0
  477. package/src/oauth/oauth-store.ts +12 -4
  478. package/src/oauth/provider-serializer.ts +8 -0
  479. package/src/oauth/seed-providers.ts +42 -0
  480. package/src/onboarding/onboarding-events-store.ts +99 -12
  481. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  482. package/src/permissions/confirmation-guardian-request.ts +2 -3
  483. package/src/permissions/question-prompter.test.ts +31 -0
  484. package/src/permissions/question-prompter.ts +2 -0
  485. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  486. package/src/persistence/attachments-store.ts +84 -44
  487. package/src/persistence/client-connection-events-store.ts +377 -0
  488. package/src/persistence/conversation-crud.ts +166 -26
  489. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  490. package/src/persistence/conversation-mode-sessions.ts +460 -0
  491. package/src/persistence/conversation-plugin-facade.ts +30 -0
  492. package/src/persistence/conversation-queries.ts +6 -19
  493. package/src/persistence/conversation-tool-surface.ts +135 -0
  494. package/src/persistence/conversation-types.test.ts +32 -0
  495. package/src/persistence/conversation-types.ts +26 -4
  496. package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +157 -32
  497. package/src/persistence/embeddings/embedding-backend-shutdown.test.ts +47 -0
  498. package/src/persistence/embeddings/embedding-backend.ts +23 -41
  499. package/src/persistence/embeddings/embedding-gemini.test.ts +17 -0
  500. package/src/persistence/embeddings/embedding-gemini.ts +3 -1
  501. package/src/persistence/embeddings/embedding-local.ts +72 -17
  502. package/src/persistence/embeddings/embedding-types.ts +0 -19
  503. package/src/persistence/llm-usage-store.ts +3 -9
  504. package/src/persistence/message-metadata.ts +25 -0
  505. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  506. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  507. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  508. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  509. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  510. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  511. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  512. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  513. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  514. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  515. package/src/persistence/schema/conversations.ts +67 -0
  516. package/src/persistence/schema/index.ts +1 -0
  517. package/src/persistence/schema/infrastructure.ts +29 -0
  518. package/src/persistence/schema/oauth.ts +1 -0
  519. package/src/persistence/steps.ts +41 -3
  520. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  521. package/src/plugin-api/conversation-turn.ts +31 -7
  522. package/src/plugin-api/effective-context-window.ts +33 -0
  523. package/src/plugin-api/index.ts +35 -6
  524. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  525. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  526. package/src/plugin-api/plugin-name-env.ts +125 -0
  527. package/src/plugin-api/resolve-credential.ts +79 -20
  528. package/src/plugin-api/vision-support.test.ts +22 -0
  529. package/src/plugin-api/vision-support.ts +17 -5
  530. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  531. package/src/plugins/active-plugin-env.ts +51 -0
  532. package/src/plugins/defaults/index.ts +24 -0
  533. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  534. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  535. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  536. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  537. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  538. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  539. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  540. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  541. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  542. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  543. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  544. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  545. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  546. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  547. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  548. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  549. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  550. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  551. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  552. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  553. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  554. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  555. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  556. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  557. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  558. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  559. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  560. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  561. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  562. package/src/plugins/defaults/memory/injectors.ts +4 -3
  563. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  564. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  565. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  566. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  567. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  568. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  569. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  570. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  571. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  572. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  573. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  574. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  575. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  576. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  577. package/src/plugins/defaults/memory/tools.ts +38 -11
  578. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  579. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  580. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  581. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  582. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  583. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  584. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  585. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  586. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  587. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  588. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  589. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  590. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  591. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  592. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  593. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  594. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  595. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  596. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  597. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  598. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  599. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  600. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  601. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  602. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  603. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  604. package/src/plugins/defaults/task-progress/package.json +14 -0
  605. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  606. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  607. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  608. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  609. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  610. package/src/prompts/delegation-gate.ts +57 -0
  611. package/src/prompts/system-prompt.ts +14 -32
  612. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  613. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  614. package/src/providers/anthropic/client.ts +10 -1
  615. package/src/providers/anthropic-gateway-shared.ts +6 -0
  616. package/src/providers/fetch-provider-catalog.ts +11 -0
  617. package/src/providers/gemini/client.ts +25 -8
  618. package/src/providers/inference/adapter-factory.ts +6 -0
  619. package/src/providers/jev/client.test.ts +270 -0
  620. package/src/providers/jev/client.ts +534 -0
  621. package/src/providers/model-catalog.ts +59 -3
  622. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  623. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  624. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  625. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  626. package/src/providers/openai/chat-completions-provider.ts +100 -29
  627. package/src/providers/openai/google-thought-signature.ts +0 -6
  628. package/src/providers/openai/responses-provider.ts +23 -1
  629. package/src/providers/platform-proxy/constants.ts +5 -0
  630. package/src/providers/ratelimit.ts +1 -3
  631. package/src/providers/search-provider-catalog.ts +24 -0
  632. package/src/providers/server-tool-pairing.ts +16 -7
  633. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  634. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  635. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  636. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  637. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  638. package/src/providers/types.ts +5 -0
  639. package/src/providers/vellum-model-routing.test.ts +1 -0
  640. package/src/routes/control.ts +7 -5
  641. package/src/runtime/AGENTS.md +6 -4
  642. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  643. package/src/runtime/actor-trust-resolver.ts +3 -3
  644. package/src/runtime/agent-wake.ts +32 -5
  645. package/src/runtime/approval-message-composer.ts +3 -69
  646. package/src/runtime/assistant-event-hub.ts +73 -75
  647. package/src/runtime/assistant-event-publish-options.ts +7 -0
  648. package/src/runtime/assistant-event-targeting.ts +79 -0
  649. package/src/runtime/assistant-stream-state.ts +13 -100
  650. package/src/runtime/auth/__tests__/middleware.test.ts +138 -0
  651. package/src/runtime/auth/middleware.ts +15 -11
  652. package/src/runtime/background-job-runner.ts +5 -5
  653. package/src/runtime/channel-retry-sweep.ts +1 -0
  654. package/src/runtime/guardian-action-message-composer.ts +8 -161
  655. package/src/runtime/guardian-action-service.ts +2 -17
  656. package/src/runtime/guardian-reply-router.ts +29 -40
  657. package/src/runtime/http-server.ts +2 -2
  658. package/src/runtime/http-types.ts +6 -0
  659. package/src/runtime/message-composer-types.ts +3 -61
  660. package/src/runtime/middleware/auth.ts +7 -1
  661. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  662. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  663. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  664. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  665. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  666. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  667. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  668. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  669. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  670. package/src/runtime/pending-interactions.ts +29 -0
  671. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  672. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  673. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  674. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  675. package/src/runtime/routes/browser-context.ts +31 -11
  676. package/src/runtime/routes/browser-routes.ts +43 -4
  677. package/src/runtime/routes/btw-routes.ts +9 -4
  678. package/src/runtime/routes/channel-route-shared.ts +1 -9
  679. package/src/runtime/routes/client-routes.ts +108 -0
  680. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  681. package/src/runtime/routes/conversation-routes.ts +125 -14
  682. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  683. package/src/runtime/routes/credential-routes.ts +1 -2
  684. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  685. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  686. package/src/runtime/routes/documents-routes.ts +51 -0
  687. package/src/runtime/routes/email-routes.ts +3 -7
  688. package/src/runtime/routes/events-routes.ts +75 -54
  689. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  690. package/src/runtime/routes/identity-routes.ts +1 -5
  691. package/src/runtime/routes/image-generation-routes.ts +8 -14
  692. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  693. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  694. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  695. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  696. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  697. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  698. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  699. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  700. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  701. package/src/runtime/routes/migration-routes.ts +73 -2
  702. package/src/runtime/routes/notification-routes.ts +10 -0
  703. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  704. package/src/runtime/routes/oauth-providers.ts +11 -0
  705. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  706. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  707. package/src/runtime/routes/plugins-routes.ts +6 -0
  708. package/src/runtime/routes/roadmap-routes.ts +47 -8
  709. package/src/runtime/routes/schedule-routes.ts +2 -0
  710. package/src/runtime/routes/secret-routes.ts +99 -119
  711. package/src/runtime/routes/settings-routes.ts +10 -4
  712. package/src/runtime/routes/watcher-routes.ts +1 -3
  713. package/src/runtime/trust-class.test.ts +7 -6
  714. package/src/runtime/trust-class.ts +9 -17
  715. package/src/runtime/verification-outbound-actions.ts +2 -2
  716. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  717. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  718. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  719. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  720. package/src/schedule/run-script.ts +2 -2
  721. package/src/schedule/scheduler.ts +47 -11
  722. package/src/schedule/tool-surface-readiness.ts +65 -0
  723. package/src/schedule/worker-control.ts +15 -5
  724. package/src/schedule/worker-mcp.ts +85 -0
  725. package/src/schedule/worker.ts +9 -31
  726. package/src/security/secure-keys.ts +74 -112
  727. package/src/skills/clawhub.ts +3 -8
  728. package/src/skills/managed-store.ts +1 -5
  729. package/src/stt/types.ts +4 -0
  730. package/src/subagent/notify.ts +54 -34
  731. package/src/subagent/parent-notification.ts +6 -0
  732. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  733. package/src/telemetry/phone-call-funnel.ts +144 -0
  734. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  735. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  736. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  737. package/src/tools/browser/browser-execution.ts +29 -60
  738. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  739. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  740. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  741. package/src/tools/client-os.ts +15 -0
  742. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  743. package/src/tools/computer-use/target.ts +42 -0
  744. package/src/tools/document/document-tool.ts +13 -18
  745. package/src/tools/execution-target.ts +14 -16
  746. package/src/tools/host-terminal/host-shell.ts +16 -7
  747. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  748. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  749. package/src/tools/network/web-fetch.ts +274 -7
  750. package/src/tools/network/web-search.ts +468 -6
  751. package/src/tools/permission-checker.ts +3 -3
  752. package/src/tools/policy-context.ts +3 -1
  753. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  754. package/src/tools/shared/shell-output.test.ts +10 -0
  755. package/src/tools/shared/shell-output.ts +14 -2
  756. package/src/tools/skills/load.ts +2 -1
  757. package/src/tools/skills/sandbox-runner.ts +24 -4
  758. package/src/tools/skills/scaffold-managed.ts +2 -1
  759. package/src/tools/skills/skill-script-runner.ts +7 -0
  760. package/src/tools/skills/skill-tool-factory.ts +15 -1
  761. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  762. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  763. package/src/tools/terminal/safe-env.ts +39 -24
  764. package/src/tools/terminal/sanitized-bash.ts +15 -2
  765. package/src/tools/terminal/shell-launch.test.ts +162 -0
  766. package/src/tools/terminal/shell.test.ts +29 -0
  767. package/src/tools/terminal/shell.ts +15 -7
  768. package/src/tools/tool-approval-handler.ts +5 -1
  769. package/src/tools/tool-types.ts +3 -3
  770. package/src/tools/types.ts +8 -2
  771. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  772. package/src/util/abort-reasons.ts +18 -26
  773. package/src/util/browser-human-verification.ts +14 -0
  774. package/src/util/canonicalize-identity.test.ts +21 -0
  775. package/src/util/canonicalize-identity.ts +16 -0
  776. package/src/util/errors.ts +13 -3
  777. package/src/util/host-process.test.ts +17 -1
  778. package/src/util/host-process.ts +24 -0
  779. package/src/util/provider-error-patterns.ts +3 -0
  780. package/src/util/unicode.ts +29 -0
  781. package/src/util/worker-pipe.test.ts +100 -0
  782. package/src/util/worker-pipe.ts +44 -0
  783. package/src/util/worker-process.ts +77 -35
  784. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  785. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  786. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  787. package/src/workspace/migrations/registry.ts +2 -0
  788. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  789. package/src/api/events/heartbeat-alert.ts +0 -20
  790. package/src/daemon/message-types/web-activity.ts +0 -65
  791. package/src/desktop/desktop-dependencies.test.ts +0 -131
  792. package/src/tools/computer-use/definitions.ts +0 -607
@@ -0,0 +1,100 @@
1
+ # Background task outcomes in live voice
2
+
3
+ The conversation's active live-voice session owns delivery of subagent updates
4
+ while the call is open. `injectMessageIntoParent` offers updates carrying
5
+ `subagentNotification` metadata to the existing `LiveVoiceSessionManager` before
6
+ enqueuing a generic parent turn. The manager only routes to its session with the
7
+ matching conversation ID. A closing session holds incoming updates for its
8
+ teardown handoff. Other conversations and fully closed calls retain the ordinary
9
+ parent delivery path.
10
+
11
+ The session keeps one outcome queue for explicit subagents and completed
12
+ interrupted-turn continuations. A newer update from the same task replaces its
13
+ pending payload and preserves its delivery state; independent tasks keep separate
14
+ entries. Success, failure, abort and mid-run updates all follow this route.
15
+ Silent/advisor instructions remain in the update so the assistant can use findings
16
+ internally. Cancelling a running continuation does not discard finished outcomes.
17
+
18
+ A user turn may read pending findings as context without consuming them. Its
19
+ completed reply playback provides evidence for the next announcement decision,
20
+ so an answer that actually covered the findings can be skipped and an unrelated
21
+ answer leaves them due. Speculative dispatches and rollbacks do not own delivery.
22
+
23
+ Announcements use the shared voice floor check: user speech, an active response,
24
+ and queued audio playback delay delivery. A hidden, tool-capable voice turn
25
+ processes the update with current conversation context and produces speech through
26
+ the normal TTS pipeline. The original notification metadata and cron attribution
27
+ travel with the turn. No synthetic user bubble is echoed. The assistant can read
28
+ additional output or resolve a blocker without starting a competing text turn.
29
+
30
+ The assistant decides whether an update adds useful information. It speaks the
31
+ new outcome before optional visual-summary tools, or outputs the internal
32
+ `[TASK_UPDATE:SILENT]` marker for routine progress or findings already heard.
33
+ The marker is stripped from speech and captions; a completed silent decision
34
+ acknowledges the update without audio.
35
+
36
+ Spoken results require forwarded reply audio and remain pending through the
37
+ estimated playback tail. Progress narration alone, empty speech synthesis, and
38
+ failed speech synthesis leave them pending. Announcement prompts carry explicit
39
+ interruption state and the latest reply with completed playback for that task.
40
+ Generated conversation text and a displayed card are not playback receipts.
41
+ Interruption state survives a progress update being replaced by its final report.
42
+ A barge-in preserves the interrupted outcome without creating another worker.
43
+ A failed launch or an
44
+ explicit stop defers retry until the next user turn or task update. Hang-up returns
45
+ undelivered updates to the ordinary conversation path after pending voice starts
46
+ and the voice turn's teardown settle, since its abort clears the parent queue.
47
+ The handoff bypasses voice routing so a replacement call cannot reclaim it during
48
+ teardown. All parent deliveries enqueue even when idle and trigger the existing
49
+ queue drain after turn finalization, so simultaneous outcomes share its processing
50
+ lock and retry behavior.
51
+
52
+ PCM is framed into at most 100ms chunks, with the first available samples emitted
53
+ immediately. Socket emission runs roughly 500ms ahead of estimated playback,
54
+ keeping prefetched synthesis from flooding the relay queue. Pacing waits stay
55
+ outside the shared outbound-frame queue and abort with the turn, so cancellation
56
+ and control frames are not delayed. Assistant audio recordings contain only
57
+ frames successfully sent to the transport. Hands-free clients reconnect on retryable
58
+ relay closes, including legacy backpressure code 4013.
59
+
60
+ The pending queue is session-local; this does not add a durable notification ledger
61
+ or client playback acknowledgements. Task records and worker output retain their
62
+ existing persistence. Completed-playback receipts mean that reply audio was sent
63
+ and its estimated playback tail elapsed without interruption; they do not prove
64
+ that the speaker was audible. The queue keeps the latest completed reply receipt
65
+ per task until the call closes. Automatic announcements follow the current
66
+ exchange; they do not modify an already-running response.
67
+
68
+ ## macOS companion QA
69
+
70
+ Start a fresh conversation from the companion and use a new investigation topic
71
+ so an earlier report cannot satisfy the request. Ask explicitly for a background
72
+ agent. Use ordinary conversational pauses; timed silence is not required.
73
+ Continuous Flux input and the diagnostics described in
74
+ [voice input diagnostics](voice-input-diagnostics.md) apply to these calls.
75
+
76
+ 1. **Ordinary completion.** Ask for a small background investigation and wait.
77
+ Confirm the task actually starts, the assistant acknowledges the request, and
78
+ useful findings are spoken without asking for them. A completion-only notice
79
+ or visual summary without an audible explanation is not sufficient.
80
+ 2. **Keep talking.** Start another investigation, then ask an unrelated question.
81
+ Include a natural pause partway through the question. The question should stay
82
+ together, the work should survive, and its results should wait for the current
83
+ exchange to finish before being announced. Repeat with an interrupted request
84
+ that continues in the background, not only an explicitly spawned agent.
85
+ 3. **Interrupt the announcement.** Speak while findings are being announced.
86
+ The assistant should yield, answer the interruption, and retain the unheard
87
+ result for appropriate delivery. Interrupting an announcement should not spawn
88
+ another copy of the investigation. Let the full report arrive after an
89
+ interrupted progress announcement; the final report must retain that delivery
90
+ state and briefly resume useful unheard findings.
91
+ 4. **Two tasks.** Request two independent investigations. Keep chatting while
92
+ they run, then leave an opening. Both useful outcomes should arrive without
93
+ overlapping audio, repetitive status announcements, or a disconnect.
94
+ 5. **Hang up before delivery.** End the call with work still running or a result
95
+ waiting to be heard. Confirm the outcome reaches the conversation afterward.
96
+
97
+ Record approximate times for requests, interruptions, missing results, and any
98
+ disconnect. Export support logs after the call. Distinguish the request being
99
+ acknowledged, the worker starting, and the result being heard; they exercise
100
+ different parts of the flow.
@@ -0,0 +1,183 @@
1
+ # Transcript mode sessions
2
+
3
+ Mode sessions are compact, conversation-owned lifecycle records for grouping
4
+ computer-use, browser, Live vision, and reserved Ambient transcript activity.
5
+ They connect runtime producers to immutable message membership while keeping
6
+ model input and stored message content unchanged.
7
+
8
+ ## API contract
9
+
10
+ `src/api/mode-session.ts` owns both shared shapes:
11
+
12
+ - `ModeSession` is immutable transcript membership: an opaque `id` and one of
13
+ `computer_use`, `browser`, `live_vision`, or `ambient`.
14
+ - `ModeSessionSummary` is the durable lifecycle record. It carries the owning
15
+ conversation, mode, timing bounds, boundary message IDs, terminal reason,
16
+ and a positive monotonic revision.
17
+
18
+ Only three statuses are stored: `active`, `completed`, and `interrupted`.
19
+ Waiting and draining are runtime presentation facts and do not add database
20
+ states. Duration is derived from recorded time bounds instead of stored as a
21
+ separate value. Active records have no terminal fields, completed records have
22
+ a confirmed end, and interrupted records may leave the end unknown. A known end
23
+ never precedes the last confirmed activity.
24
+
25
+ ## Persistence
26
+
27
+ Migration 381 creates `conversation_mode_sessions`. Each row belongs to one
28
+ conversation through an `ON DELETE CASCADE` foreign key. The
29
+ `(conversation_id, status)` index supports conversation-scoped active reads,
30
+ while the primary key supports bounded ID batches for transcript pages.
31
+
32
+ `conversation-mode-sessions.ts` is the only lifecycle writer. Begin creates
33
+ revision 1. Activity, boundary, revision-only, and terminal updates require the
34
+ caller's expected revision and run in an immediate transaction. Each accepted
35
+ write increments the revision. A stale revision, missing record, or terminal
36
+ record is returned as an expected write outcome. Terminal rows cannot return
37
+ to active or change to a different terminal result. Repeating the same terminal
38
+ outcome is an accepted no-op and does not advance the revision.
39
+
40
+ The first included boundary can precede source activation when a later action
41
+ claims assistant content already persisted in the same logical turn. Last
42
+ activity never moves backward. An interruption may have an unknown end, so
43
+ `endedAt` remains nullable and `lastActivityAt` remains the last confirmed
44
+ time.
45
+
46
+ ## Startup recovery
47
+
48
+ After database migrations succeed, the assistant marks every leftover active
49
+ record interrupted before database readiness is published. Recovery increments
50
+ the revision, preserves activity and message boundaries, records
51
+ `assistant_restarted`, and leaves `endedAt` null. It runs on every boot rather
52
+ than as a checkpointed migration.
53
+
54
+ If this auxiliary recovery fails, ordinary database readiness succeeds and
55
+ tracking stays unavailable for that boot. History omits unrecovered active
56
+ descriptors while preserving terminal records and message stamps. A subsequent
57
+ boot retries through the same startup path. Required migration failures still
58
+ block database readiness.
59
+
60
+ Runtime source generations and structural question associations are owned by
61
+ the producer and message-membership integration. Those process-local facts are
62
+ not reconstructed after restart. Resumed mode work uses a fresh session ID and
63
+ callbacks from retired source generations are rejected.
64
+
65
+ New producer admissions are controlled by the default-off `session-groups`
66
+ assistant feature flag. Browser and computer-use actions still execute when the
67
+ flag is off, but they do not activate or claim a mode session. Existing owners
68
+ and terminal cleanup continue to drain so disabling the flag cannot strand
69
+ accepted work. Live voice snapshots the flag when its socket session is
70
+ created: disabled sessions retain the existing camera transport without
71
+ advertising or preparing tracked sight-session epochs.
72
+
73
+ The same flag controls activity-label wording at the shared rendering
74
+ boundary. Disabled views retain their existing labels and phase grouping.
75
+
76
+ The migration, startup recovery, persisted message stamps, and history reads
77
+ remain available while the flag is off. This keeps records made by an enabled
78
+ cohort compatible with rollback and later re-enablement.
79
+
80
+ ## Runtime ownership
81
+
82
+ Each live `Conversation` owns one `ConversationModeSessionCoordinator`.
83
+ Computer-use, browser, and camera producers activate a source with a stable
84
+ source identity and guarded generation. The first accepted mode action claims
85
+ the logical turn. The coordinator backfills rows already persisted for that
86
+ turn, then stamps later user, assistant, and tool-result rows as they are
87
+ inserted. A turn keeps its first owner when another mode acts inside it.
88
+
89
+ Source eligibility and captured turn ownership end separately. Ending or
90
+ resetting a source prevents future claims immediately, but a captured turn
91
+ keeps its owner until its final output is persisted. Ordinary computer-use and
92
+ browser sources finish when the turn settles. Camera sources use a source
93
+ lifetime plus a synthetic run turn, so accepted frame writes and captured
94
+ voice turns form a settlement barrier before terminal persistence. A camera
95
+ epoch snapshots ownership before asynchronous frame persistence; stale epochs
96
+ cannot join a later run.
97
+
98
+ Tracking failures are logged and isolated from authorized actions and accepted
99
+ content persistence. They never repeat the underlying action. A refused camera
100
+ start clears client negotiation and falls back to untracked frames without
101
+ revoking consent or ending voice. Individual stale-frame rejection does not
102
+ use this fallback.
103
+
104
+ Server admission is the camera boundary: an admitted frame can finish saving
105
+ after stop, but a device-side capture delivered after its run ends is rejected
106
+ and reclaimed. Reconnect pauses capture delivery and announces a fresh run
107
+ after replacement readiness. In-flight uploads and native samples retain their
108
+ captured ownership and cannot join the replacement run.
109
+
110
+ The coordinator stores only `active`, `completed`, or `interrupted`. Waiting
111
+ and finishing are revisioned runtime hints. Terminal dispositions remain
112
+ pending in memory until all captured turns release, then one idempotent
113
+ compare-and-swap update records the terminal outcome. A successful turn with
114
+ no explicit producer end also finalizes, allowing later work from the same
115
+ resource generation to mint a new session.
116
+
117
+ Regular reload, TTL, LRU, and memory-pressure eviction retain a conversation
118
+ while it has an eligible mode source, captured owner, structural association,
119
+ or unsettled terminal disposition. This preserves accepted camera output and
120
+ exact structural continuation in the coordinator that admitted them. A long
121
+ structural wait therefore remains resident until it is consumed, invalidated,
122
+ or its source retires. Explicit conversation deletion and process shutdown
123
+ keep their existing teardown semantics; startup recovery interrupts any
124
+ durable active record left by process exit.
125
+
126
+ Invalidating the final structural association retires a turn-lifetime source
127
+ and lets accepted output settle. Successful abandoned waits end at their last
128
+ confirmed activity with reason `structural_wait_abandoned`; cleanup does not
129
+ extend their duration. Source-lifetime camera runs remain eligible. A retired
130
+ disposition that cannot persist its terminal update releases residency and
131
+ invalidates history without claiming completion. An active record without live
132
+ ownership has no descriptor and renders as unavailable.
133
+
134
+ ## Structural continuation
135
+
136
+ Structural waits associate an exact response identifier with the owned turn.
137
+ Ordinary `ask_question` awaits its resolver inside that turn, so its answer
138
+ resumes the existing owner. A blocking UI surface can accept a new turn: its
139
+ matching response consumes the association before surface cleanup and retains
140
+ the session ID. Rejected, stale, unrelated, or cross-conversation responses
141
+ cannot inherit ownership. Dismissal invalidates the association. Ordinary prose
142
+ and an idle reusable resource do not continue a session; later mode work starts
143
+ a fresh ID and duration.
144
+
145
+ Structural associations intentionally do not survive restart. Startup recovery
146
+ terminalizes the old record, and a late response remains part of the existing
147
+ interaction flow without reopening that record.
148
+
149
+ ## Transcript and synchronization
150
+
151
+ `modeSession` is optional message metadata and is stripped from caller-supplied
152
+ metadata. Membership is added only after a row insert succeeds. Text and
153
+ thinking deltas carry no repeated ownership field; row-boundary events carry
154
+ the current owner, and the normal conversation-messages invalidation refetches
155
+ retrospective stamps.
156
+
157
+ History returns one batched `modeSessions` descriptor list beside messages.
158
+ The query resolves records for page members and accepts bounded explicit IDs
159
+ for groups already loaded outside the newest page. Consolidation never crosses
160
+ a stamped/unstamped or different-owner boundary. Same-owner folds retain row
161
+ aliases and `modeSessionActivity.firstAt`/`lastAt`, so donor timestamps survive
162
+ history compaction.
163
+
164
+ Message deletion and retry paths recompute replaceable first/last display
165
+ boundaries for every affected conversation session. They preserve status,
166
+ terminal time, reason, and confirmed activity, and increment the revision.
167
+ Deleting every owned row leaves the lifecycle record available for audit but
168
+ produces no transcript container.
169
+
170
+ All membership, lifecycle, boundary, and runtime-hint changes reuse the
171
+ conversation-messages `sync_changed` tag. Clients merge descriptors by
172
+ revision in the history query cache and refetch through the existing history
173
+ reconciliation path. Explicit stale revisions cannot replace newer cached
174
+ descriptors; an omitted descriptor remains unavailable. There is no separate
175
+ descriptor store, session event stream, or polling endpoint.
176
+
177
+ The transcript keeps lightweight segment identities for stable disclosure keys,
178
+ not duplicate message bodies. Closed children unmount, and eligible mounted
179
+ headers share one coarse clock that pauses while the app is hidden. No header
180
+ visibility observer is required. The clock reads initial desktop visibility
181
+ from the existing attention state on its coarse tick, without publishing a
182
+ shared lifecycle edge. With grouping disabled, the load-older guard uses the
183
+ ordinary flat-transcript behavior and skips grouped DOM scans.
@@ -0,0 +1,58 @@
1
+ # Voice input diagnostics
2
+
3
+ Hands-free voice records signal measurements in the ordinary support log export. This includes calls started from the macOS companion. These diagnostics do not change capture constraints, speech thresholds, interruption timing, or task cancellation. They add no PCM recordings, transcript text, microphone labels, or device identifiers.
4
+
5
+ ## Collect a reproduction
6
+
7
+ Use the same microphone, output device, volume, and companion surface as the failing call. Note whether output is through headphones or speakers.
8
+
9
+ 1. Start a companion voice call. Stay quiet for about 20 seconds to capture the room baseline.
10
+ 2. Ask for an explanation long enough to produce sustained speech. Stay quiet while it answers; note the time of any unexpected cutoff.
11
+ 3. Ask it to start a background investigation. Stay quiet during startup; note missing acknowledgements or cancelled starts.
12
+ 4. Deliberately interrupt an answer with a short spoken question. This provides a real interruption to compare against the unexplained cutoffs.
13
+ 5. End the call and export logs through the existing feedback log export.
14
+
15
+ Keep the initial reproduction on the original audio setup. A second call using headphones, or using the built-in microphone instead of a headset, can isolate a route-dependent problem after the first capture.
16
+
17
+ ## Client records
18
+
19
+ `[live-voice-input]` entries are serialized JSON in `electron-main-logs.txt`. Electron collects them from every renderer, so export does not depend on which window hosts the companion call. The same events appear as `voice_input` in the web diagnostics lifecycle ring.
20
+
21
+ - `capture_started` and `input_switched` report applied echo cancellation, noise suppression, gain control, channel count, and track/context/output sample rates. `null` means the browser did not report a setting. An explicit device selection reports whether it matched, without logging either device identifier.
22
+ - `capture_mute`, `capture_unmute`, `capture_ended`, and `capture_statechange` expose track and audio-context transitions. Listeners retire with the old stream on a device switch or stop.
23
+ - `session_ready` links the client `captureId` to the server `sessionId`. Capture can open before the server is ready, so earlier entries have a null session ID. `entry` identifies the launch surface when supplied.
24
+ - `speech_started`, `turn_cancelled`, and `utterance_discarded` report whether playback was active, held, or actually resumed after an unusable utterance.
25
+
26
+ ## Server records
27
+
28
+ Search the assistant log for `Live voice input diagnostics` or `voice_input_`. All records include session, input-turn, assistant-turn, and speech-generation identifiers where available, plus the effective threshold, room floor, echo configuration, playback estimate, and foreground-task state.
29
+
30
+ - `voice_input_window` summarizes each approximately five-second arrival window: received audio duration, zero-signal duration, duration-weighted mean amplitude, minimum/maximum chunk mean amplitude, classified speech/silence/echo duration, and maximum arrival gap. Window emission requires incoming audio; `voice_input_closed` flushes a partial window.
31
+ - The same window includes `sttSubmittedChunks`, `sttSubmittedAudioMs`, and `sttMaxSubmissionGapMs`. These measure handoff to the transcription adapter, not provider receipt. A probe can buffer audio before submission, and the final chunk of an arrival window can be submitted in the next window. Compare sustained windows to identify input that arrives normally but stops reaching transcription.
32
+ - `voice_input_provider_turn_start` and `voice_input_provider_turn_end` record provider boundaries alongside local VAD state. End records include confidence, the provider's `trigger` (`model`, `timeout`, or `manual`), decoded audio position in seconds from stream start, and elapsed time since locally detected speech. Missing provider fields are null. These records describe received decisions, including ones ignored as stale or after fallback; correlate with the turn metrics to identify the boundary that committed the request.
33
+ - `voice_input_microphone_gate_changed` records when the transcription audio gate closes or reopens. `microphoneGateClosed` is included with the other input diagnostics. Closing replaces room audio with silence; it does not commit a transcript or stop local energy detection.
34
+ - `voice_input_barge_in_armed`, `voice_input_barge_in_fired`, and `voice_input_barge_in_expired` distinguish a candidate from an accepted or expired local interruption on providers without native turn detection. Fired events include accumulated speech duration, guard resets, and a bounded trace of the preceding two seconds (at most 40 input chunks).
35
+ - `voice_input_speech_started` includes `source: "provider"` when a Flux `StartOfTurn` triggers the client speech signal and any active-turn cancellation. `source: "local"` identifies local onset or a confirmed pre-dial guard replay; guards completed during ordinary streaming use `voice_input_barge_in_fired`.
36
+ - Each trace entry captures the amplitude and gate/echo state before classification, and classified durations, remaining probe buffer, and correlation result after classification. The classifier can hold a probe and release multiple older chunks together, so a row's classified duration can differ from its received duration. A null correlation means no comparison ran, not a measured zero correlation.
37
+ - `voice_input_speech_routed` binds the speech generation to the input turn that actually receives its audio. At onset, the prior spent cycle may still be installed, so use this routing record for correlation rather than assuming the onset's `inputTurnId` is the new utterance.
38
+ - `voice_input_transcript` reports committed character count and provider, never words. `voice_input_utterance_discarded` reports the discard reason and remaining transcript/partial state.
39
+
40
+ Amplitude uses the classifier's PCM16 mean-absolute scale, not the client's normalized UI meter. Playback timing is the server's estimate, not proof that sound was heard. An empty transcript is evidence of no usable transcription; it does not establish that the person was silent or that the detector was wrong.
41
+
42
+ These are local diagnostic logs, not new analytics events or voice protocol fields. Existing log retention and export apply.
43
+
44
+ ## Flux pause handling
45
+
46
+ When Flux owns turn detection, microphone audio passes through for one second of received audio after the last locally detected speech. After that, the input gate replaces room audio with digital silence while the microphone and local energy detector keep listening. The cutoff is measured in PCM samples, so network batching does not change it. Flux continues receiving elapsed silence and owns the turn boundary; closing this gate does not release the question.
47
+
48
+ While closed, the gate holds the most recent 200 ms before deciding whether to send those samples as silence or as the lead-in to resumed speech. On speech onset, it forwards that lead-in and the new speech exactly once. The provider feed can lag input by up to 200 ms while idle, then catches up on speech onset. Confirmed playback echo is replaced with silence before entering this buffer. Idle audio is not accumulated into request recordings. Manual capture and locally endpointed providers retain their existing routing.
49
+
50
+ Flux `StartOfTurn` owns hands-free interruption, independently of the provider end-of-turn setting. Above-threshold microphone audio alone cannot pause playback, cancel a reply, or trigger a speech-start screenshot. A provider turn start emits `speech_started` immediately, without the local sustained-speech delay, and cancels an active reply through the shared interruption path. Duplicate or older provider turn indices are ignored within a stream. Other providers retain the local guard. This waits for provider recognition and does not guarantee rejection of playback echo or background speech that Flux transcribes.
51
+
52
+ Managed Flux maps provider turn indices into one continuous sequence across transparent relay reconnects. A new relay connection restarting at zero therefore cannot make fresh speech look like a duplicate. If connection setup falls back to a provider without native turn detection, the suppressed local onset is replayed through the local interruption path, preserving its sustained-speech guard result even if the utterance ended during the dial.
53
+
54
+ The gate shares the existing adaptive energy classifier. Quiet speech within the one-second tail passes through; quiet onset in the 200 ms buffer is preserved if followed by detected speech. Sustained speech entirely below the local threshold can be suppressed, and noise classified as speech can reopen the gate. This limits background exposure rather than guaranteeing that every false transcript is eliminated.
55
+
56
+ For a focused reproduction, say an unfinished question such as "Did they help invent", pause briefly, then complete it with "ChatGPT". Repeat with a shorter and a longer pause, plus a complete short question. Note when the assistant starts answering and export the logs. Check that STT submissions continue during the pause, then use the provider end event's time, confidence, and trigger to distinguish model endpointing from a timeout. Tests verify audio delivery and event handling; they do not establish how Flux will segment real speech.
57
+
58
+ For the idle gate, stay silent and type for 20 seconds, then ask a question with a pause longer than one second. Repeat with a quiet voice and with a deliberate interruption during an answer. Check for phantom requests, a clipped first syllable, split questions, and missed interruptions. Correlate those observations with gate transitions and provider turn events.
@@ -27,7 +27,8 @@
27
27
  "./reactions": "./src/reactions.ts",
28
28
  "./guardian-requests": "./src/guardian-requests.ts",
29
29
  "./platform-credential": "./src/platform-credential.ts",
30
- "./plan-credit": "./src/plan-credit.ts"
30
+ "./plan-credit": "./src/plan-credit.ts",
31
+ "./integration-categories": "./src/integration-categories.ts"
31
32
  },
32
33
  "scripts": {
33
34
  "typecheck": "bunx tsc --noEmit",
@@ -63,6 +63,7 @@ const FIXTURES: Record<string, string> = {
63
63
  "Perplexity API Key": `pplx-${filler(40)}`,
64
64
  "Tavily API Key": `tvly-${filler(20)}`,
65
65
  "Firecrawl API Key": `fc-${filler(20)}`,
66
+ "Resend API Key": `re_${filler(8)}_${filler(24)}`,
66
67
  };
67
68
 
68
69
  describe("subpath export", () => {
@@ -29,6 +29,7 @@ export * from "./trust-rules.js";
29
29
  export * from "./ingress.js";
30
30
  export * from "./no-response.js";
31
31
  export * from "./plan-credit.js";
32
+ export * from "./integration-categories.js";
32
33
  export * from "./platform-credential.js";
33
34
  export * from "./remote-web-pairing.js";
34
35
  export * from "./twilio-ingress.js";
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Category vocabulary for the integrations catalog, shared by the assistant
3
+ * daemon (which stamps it on OAuth providers and marketplace MCP entries) and
4
+ * the clients (which draw the category filter from it).
5
+ *
6
+ * The order here is the order clients show the categories in. The slug is the
7
+ * wire value; the user-facing label is the client's, so it can be translated.
8
+ */
9
+ export const INTEGRATION_CATEGORIES = [
10
+ "productivity",
11
+ "communication",
12
+ "meetings",
13
+ "sales",
14
+ "marketing",
15
+ "finance",
16
+ "commerce",
17
+ "engineering",
18
+ "knowledge",
19
+ "recruiting",
20
+ ] as const;
21
+
22
+ export type IntegrationCategory = (typeof INTEGRATION_CATEGORIES)[number];
23
+
24
+ const INTEGRATION_CATEGORY_SET: ReadonlySet<string> = new Set(
25
+ INTEGRATION_CATEGORIES,
26
+ );
27
+
28
+ /**
29
+ * Whether `value` names a category this build knows. A client reading a newer
30
+ * catalog treats an unknown slug as uncategorized rather than failing.
31
+ */
32
+ export function isIntegrationCategory(
33
+ value: unknown,
34
+ ): value is IntegrationCategory {
35
+ return typeof value === "string" && INTEGRATION_CATEGORY_SET.has(value);
36
+ }
@@ -230,6 +230,9 @@ export const PREFIX_PATTERNS: SecretPrefixPattern[] = [
230
230
 
231
231
  // -- Firecrawl --
232
232
  { label: "Firecrawl API Key", regex: /fc-[A-Za-z0-9]{20,}/ },
233
+ // Resend documents only the `re_` prefix; the segment shape is the observed
234
+ // key format. Strict on purpose: bare `re_` prefixes ordinary identifiers.
235
+ { label: "Resend API Key", regex: /re_[A-Za-z0-9]{8}_[A-Za-z0-9]{24,}/ },
233
236
  ];
234
237
 
235
238
  /**
@@ -362,7 +365,7 @@ export function isPlaceholderValue(value: string): boolean {
362
365
  // Strip known prefixes to isolate the variable part
363
366
  const variablePart = value
364
367
  .replace(
365
- /^(?:AKIA|gh[pousr]_|github_pat_|glpat-|sk_live_|rk_live_|xoxb-|xoxp-|xapp-|sk-ant-|sk-proj-|sk-or-v1-|AIza|GOCSPX-|SK|SG\.|npm_|pypi-|key-|lin_api_|ntn_|fw_|pplx-|-----BEGIN [A-Z ]*PRIVATE KEY-----)/,
368
+ /^(?:AKIA|gh[pousr]_|github_pat_|glpat-|sk_live_|rk_live_|xoxb-|xoxp-|xapp-|sk-ant-|sk-proj-|sk-or-v1-|AIza|GOCSPX-|SK|SG\.|npm_|pypi-|key-|lin_api_|ntn_|fw_|pplx-|re_|-----BEGIN [A-Z ]*PRIVATE KEY-----)/,
366
369
  "",
367
370
  )
368
371
  .replace(/[^A-Za-z0-9]/g, "");
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
2
2
 
3
3
  import {
4
4
  buildShellInvocation,
5
+ buildShellSpawnFlags,
5
6
  pathListDelimiter,
6
7
  prependUniquePathEntries,
7
8
  } from "./shell.js";
@@ -38,6 +39,26 @@ describe("buildShellInvocation", () => {
38
39
  });
39
40
  });
40
41
 
42
+ describe("buildShellSpawnFlags", () => {
43
+ test("creates a POSIX process group and hides Windows consoles", () => {
44
+ expect(buildShellSpawnFlags("linux")).toEqual({
45
+ detached: true,
46
+ windowsHide: true,
47
+ });
48
+ expect(buildShellSpawnFlags("darwin")).toEqual({
49
+ detached: true,
50
+ windowsHide: true,
51
+ });
52
+ });
53
+
54
+ test("does not detach Windows children that use piped stdio", () => {
55
+ expect(buildShellSpawnFlags("win32")).toEqual({
56
+ detached: false,
57
+ windowsHide: true,
58
+ });
59
+ });
60
+ });
61
+
41
62
  describe("path list handling", () => {
42
63
  test("uses the platform delimiter", () => {
43
64
  expect(pathListDelimiter("win32")).toBe(";");
@@ -5,6 +5,11 @@ export interface ShellInvocation {
5
5
  args: string[];
6
6
  }
7
7
 
8
+ export interface ShellSpawnFlags {
9
+ detached: boolean;
10
+ windowsHide: true;
11
+ }
12
+
8
13
  const WINDOWS_UTF8_PREAMBLE =
9
14
  "try { [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch {}; " +
10
15
  "$OutputEncoding = [System.Text.Encoding]::UTF8; " +
@@ -43,6 +48,25 @@ export function buildShellInvocation(
43
48
  return { command: "bash", args: ["-c", "--", command] };
44
49
  }
45
50
 
51
+ /**
52
+ * Spawn flags for assistant-owned shell children (sandbox bash, local
53
+ * host_bash fallback, sanitized CLI bash, skill runners, scheduled scripts).
54
+ *
55
+ * POSIX uses a new process group so timeout/abort can SIGKILL the tree via
56
+ * `-pid`. Windows process trees are torn down with `taskkill /T`, which does
57
+ * not need a detached process. Combining `DETACHED_PROCESS`,
58
+ * `CREATE_NO_WINDOW`, and piped stdio on Windows can emit `close` with exit
59
+ * 0 and empty pipes without running the encoded command.
60
+ */
61
+ export function buildShellSpawnFlags(
62
+ hostPlatform: NodeJS.Platform = process.platform,
63
+ ): ShellSpawnFlags {
64
+ return {
65
+ detached: hostPlatform !== "win32",
66
+ windowsHide: true,
67
+ };
68
+ }
69
+
46
70
  export function pathListDelimiter(
47
71
  hostPlatform: NodeJS.Platform = process.platform,
48
72
  ): string {
@@ -27,7 +27,8 @@
27
27
  "./reactions": "./src/reactions.ts",
28
28
  "./guardian-requests": "./src/guardian-requests.ts",
29
29
  "./platform-credential": "./src/platform-credential.ts",
30
- "./plan-credit": "./src/plan-credit.ts"
30
+ "./plan-credit": "./src/plan-credit.ts",
31
+ "./integration-categories": "./src/integration-categories.ts"
31
32
  },
32
33
  "scripts": {
33
34
  "typecheck": "bunx tsc --noEmit",
@@ -63,6 +63,7 @@ const FIXTURES: Record<string, string> = {
63
63
  "Perplexity API Key": `pplx-${filler(40)}`,
64
64
  "Tavily API Key": `tvly-${filler(20)}`,
65
65
  "Firecrawl API Key": `fc-${filler(20)}`,
66
+ "Resend API Key": `re_${filler(8)}_${filler(24)}`,
66
67
  };
67
68
 
68
69
  describe("subpath export", () => {
@@ -29,6 +29,7 @@ export * from "./trust-rules.js";
29
29
  export * from "./ingress.js";
30
30
  export * from "./no-response.js";
31
31
  export * from "./plan-credit.js";
32
+ export * from "./integration-categories.js";
32
33
  export * from "./platform-credential.js";
33
34
  export * from "./remote-web-pairing.js";
34
35
  export * from "./twilio-ingress.js";
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Category vocabulary for the integrations catalog, shared by the assistant
3
+ * daemon (which stamps it on OAuth providers and marketplace MCP entries) and
4
+ * the clients (which draw the category filter from it).
5
+ *
6
+ * The order here is the order clients show the categories in. The slug is the
7
+ * wire value; the user-facing label is the client's, so it can be translated.
8
+ */
9
+ export const INTEGRATION_CATEGORIES = [
10
+ "productivity",
11
+ "communication",
12
+ "meetings",
13
+ "sales",
14
+ "marketing",
15
+ "finance",
16
+ "commerce",
17
+ "engineering",
18
+ "knowledge",
19
+ "recruiting",
20
+ ] as const;
21
+
22
+ export type IntegrationCategory = (typeof INTEGRATION_CATEGORIES)[number];
23
+
24
+ const INTEGRATION_CATEGORY_SET: ReadonlySet<string> = new Set(
25
+ INTEGRATION_CATEGORIES,
26
+ );
27
+
28
+ /**
29
+ * Whether `value` names a category this build knows. A client reading a newer
30
+ * catalog treats an unknown slug as uncategorized rather than failing.
31
+ */
32
+ export function isIntegrationCategory(
33
+ value: unknown,
34
+ ): value is IntegrationCategory {
35
+ return typeof value === "string" && INTEGRATION_CATEGORY_SET.has(value);
36
+ }
@@ -230,6 +230,9 @@ export const PREFIX_PATTERNS: SecretPrefixPattern[] = [
230
230
 
231
231
  // -- Firecrawl --
232
232
  { label: "Firecrawl API Key", regex: /fc-[A-Za-z0-9]{20,}/ },
233
+ // Resend documents only the `re_` prefix; the segment shape is the observed
234
+ // key format. Strict on purpose: bare `re_` prefixes ordinary identifiers.
235
+ { label: "Resend API Key", regex: /re_[A-Za-z0-9]{8}_[A-Za-z0-9]{24,}/ },
233
236
  ];
234
237
 
235
238
  /**
@@ -362,7 +365,7 @@ export function isPlaceholderValue(value: string): boolean {
362
365
  // Strip known prefixes to isolate the variable part
363
366
  const variablePart = value
364
367
  .replace(
365
- /^(?:AKIA|gh[pousr]_|github_pat_|glpat-|sk_live_|rk_live_|xoxb-|xoxp-|xapp-|sk-ant-|sk-proj-|sk-or-v1-|AIza|GOCSPX-|SK|SG\.|npm_|pypi-|key-|lin_api_|ntn_|fw_|pplx-|-----BEGIN [A-Z ]*PRIVATE KEY-----)/,
368
+ /^(?:AKIA|gh[pousr]_|github_pat_|glpat-|sk_live_|rk_live_|xoxb-|xoxp-|xapp-|sk-ant-|sk-proj-|sk-or-v1-|AIza|GOCSPX-|SK|SG\.|npm_|pypi-|key-|lin_api_|ntn_|fw_|pplx-|re_|-----BEGIN [A-Z ]*PRIVATE KEY-----)/,
366
369
  "",
367
370
  )
368
371
  .replace(/[^A-Za-z0-9]/g, "");
@@ -43,6 +43,26 @@ export type GatewayLogsTailRouteParams = z.infer<
43
43
  typeof GatewayLogsTailRouteParamsSchema
44
44
  >;
45
45
 
46
+ /**
47
+ * `gateway_debug_export`: a consistent copy of the gateway database plus its
48
+ * recent log files, as one gzipped tar, for a debug bundle. No params.
49
+ */
50
+ export const GatewayDebugExportIpcParamsSchema = z
51
+ .object({})
52
+ .strict()
53
+ .default({});
54
+
55
+ export const GatewayDebugExportIpcResponseSchema = z.object({
56
+ ok: z.literal(true),
57
+ /** `tar.gz` bytes, base64. Contains gateway.sqlite and gateway-logs/. */
58
+ archive_base64: z.string(),
59
+ size_bytes: z.number().int().nonnegative(),
60
+ });
61
+
62
+ export type GatewayDebugExportIpcResponse = z.infer<
63
+ typeof GatewayDebugExportIpcResponseSchema
64
+ >;
65
+
46
66
  export const GatewayLogsTailIpcResponseSchema = z.object({
47
67
  lines: z.array(z.record(z.string(), z.unknown())),
48
68
  truncated: z.boolean(),
@@ -83,8 +83,14 @@ export const SourceMetadataSchema = z
83
83
  clientTimezone: z.string().optional(),
84
84
  /** Channel command intent (e.g. Telegram /start). */
85
85
  commandIntent: CommandIntentSchema.optional(),
86
- /** Slack-specific: whether the bot was @-mentioned. */
87
- slackBotMentioned: z.boolean().optional(),
86
+ /**
87
+ * Whether the message addresses the assistant by name, an @-mention on
88
+ * the platforms that have one. Stated by the channel's normalizer only
89
+ * where it proved the answer; absent means "not established". Read to
90
+ * decide whether a reply is expected before any text exists, never to
91
+ * admit or deny: admission is the gate's job, upstream of this.
92
+ */
93
+ botMentioned: z.boolean().optional(),
88
94
  /**
89
95
  * Slack-specific: the `edited.ts` float-string from a `message_changed`
90
96
  * event. Used by the daemon to reject out-of-order edit deliveries whose