@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (859) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +5 -1
  3. package/ARCHITECTURE.md +20 -21
  4. package/Dockerfile +7 -7
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +11 -2
  7. package/docs/architecture/turn-actor.md +15 -8
  8. package/docs/credential-execution-service.md +1 -1
  9. package/docs/desktop-browser-cli.md +35 -0
  10. package/docs/guardian-request-flow.md +16 -0
  11. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  12. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  13. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  14. package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
  15. package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
  16. package/openapi.yaml +564 -111
  17. package/package.json +7 -4
  18. package/scripts/bundled-plugin-packages.ts +154 -0
  19. package/scripts/generate-bundled-plugin-packages.ts +29 -0
  20. package/scripts/postinstall.ts +33 -0
  21. package/scripts/smoke-desktop-browser-cli.ts +280 -0
  22. package/scripts/smoke-desktop-window-theme.ts +186 -0
  23. package/scripts/sync-llm-catalog.ts +3 -0
  24. package/scripts/test.ts +15 -13
  25. package/scripts/voice-ttft-spike.ts +2 -2
  26. package/src/__tests__/agent-loop.test.ts +124 -0
  27. package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
  28. package/src/__tests__/anthropic-provider.test.ts +0 -63
  29. package/src/__tests__/app-compiler.test.ts +6 -0
  30. package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
  31. package/src/__tests__/app-dir-path-guard.test.ts +1 -3
  32. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  33. package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
  34. package/src/__tests__/assistant-event-hub.test.ts +44 -0
  35. package/src/__tests__/attachments-store.test.ts +22 -3
  36. package/src/__tests__/channel-approval.test.ts +9 -14
  37. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  38. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  39. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  40. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  41. package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
  42. package/src/__tests__/computer-use-tools.test.ts +19 -7
  43. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  44. package/src/__tests__/config-schema.test.ts +29 -30
  45. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  46. package/src/__tests__/config-watcher.test.ts +23 -8
  47. package/src/__tests__/connection-model-compat.test.ts +2 -2
  48. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  49. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  50. package/src/__tests__/conversation-agent-loop.test.ts +257 -6
  51. package/src/__tests__/conversation-attachments.test.ts +106 -0
  52. package/src/__tests__/conversation-error.test.ts +4 -1
  53. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  54. package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
  55. package/src/__tests__/conversation-pairing.test.ts +26 -0
  56. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
  57. package/src/__tests__/conversation-queue.test.ts +48 -207
  58. package/src/__tests__/conversation-retry-discard.test.ts +54 -1
  59. package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
  60. package/src/__tests__/conversation-skill-tools.test.ts +27 -0
  61. package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
  62. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
  63. package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
  64. package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
  65. package/src/__tests__/conversation-title-service.test.ts +14 -10
  66. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
  67. package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
  68. package/src/__tests__/credential-execution-client.test.ts +70 -53
  69. package/src/__tests__/credential-record-retire.test.ts +171 -1
  70. package/src/__tests__/credential-routes.test.ts +250 -3
  71. package/src/__tests__/credential-security-invariants.test.ts +5 -3
  72. package/src/__tests__/cu-unified-flow.test.ts +14 -4
  73. package/src/__tests__/db-conversation-tool-surface.test.ts +144 -0
  74. package/src/__tests__/document-sync-tags.test.ts +25 -0
  75. package/src/__tests__/drain-kick-guard.test.ts +0 -2
  76. package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
  77. package/src/__tests__/events-tail-route.test.ts +33 -0
  78. package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
  79. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  80. package/src/__tests__/git-porcelain.test.ts +47 -0
  81. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  82. package/src/__tests__/heartbeat-service.test.ts +56 -40
  83. package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
  84. package/src/__tests__/host-bash-proxy.test.ts +26 -45
  85. package/src/__tests__/host-browser-proxy.test.ts +6 -17
  86. package/src/__tests__/host-cu-proxy.test.ts +712 -18
  87. package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
  88. package/src/__tests__/host-file-proxy.test.ts +11 -23
  89. package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
  90. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  91. package/src/__tests__/list-all-apps.test.ts +40 -8
  92. package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
  93. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  94. package/src/__tests__/llm-catalog-parity.test.ts +51 -3
  95. package/src/__tests__/managed-store.test.ts +121 -0
  96. package/src/__tests__/mcp-auth-routes.test.ts +104 -8
  97. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  98. package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
  99. package/src/__tests__/mcp-health-check.test.ts +40 -43
  100. package/src/__tests__/mcp-list-plugin-servers.test.ts +85 -42
  101. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  102. package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
  103. package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
  104. package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
  105. package/src/__tests__/messaging-send-tool.test.ts +355 -548
  106. package/src/__tests__/model-intents.test.ts +3 -0
  107. package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
  108. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  109. package/src/__tests__/oauth-commands-routes.test.ts +282 -101
  110. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  111. package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
  112. package/src/__tests__/oauth-provider-serializer.test.ts +23 -0
  113. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  114. package/src/__tests__/openai-provider.test.ts +1 -0
  115. package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
  116. package/src/__tests__/persist-media-references.test.ts +50 -0
  117. package/src/__tests__/platform-callback-registration.test.ts +8 -16
  118. package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +13 -0
  120. package/src/__tests__/qdrant-manager.test.ts +132 -0
  121. package/src/__tests__/resolve-app.test.ts +135 -0
  122. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  123. package/src/__tests__/scaffold-managed-skill-tool.test.ts +317 -6
  124. package/src/__tests__/schedule-routes.test.ts +20 -0
  125. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  126. package/src/__tests__/script-proxy-certs.test.ts +35 -0
  127. package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
  128. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  129. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  130. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  131. package/src/__tests__/secure-keys.test.ts +7 -3
  132. package/src/__tests__/service-account-token.test.ts +247 -0
  133. package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
  134. package/src/__tests__/skill-feature-flags.test.ts +2 -2
  135. package/src/__tests__/skills.test.ts +20 -4
  136. package/src/__tests__/source-fingerprint.test.ts +184 -0
  137. package/src/__tests__/subagent-tool-gate-mode.test.ts +239 -3
  138. package/src/__tests__/subagent-tools.test.ts +3 -2
  139. package/src/__tests__/telephony-tts-guard.test.ts +38 -3
  140. package/src/__tests__/terminal-tools.test.ts +8 -0
  141. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  142. package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
  143. package/src/__tests__/ui-channel-variants.test.ts +1 -56
  144. package/src/__tests__/unicode.test.ts +49 -0
  145. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  146. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
  147. package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
  148. package/src/__tests__/workspace-git-service.test.ts +26 -0
  149. package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
  150. package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
  151. package/src/acp/__tests__/acp-credentials.test.ts +13 -0
  152. package/src/acp/__tests__/agent-process.test.ts +258 -4
  153. package/src/acp/__tests__/auth-required.test.ts +96 -0
  154. package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
  155. package/src/acp/__tests__/client-handler.test.ts +90 -2
  156. package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
  157. package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
  158. package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
  159. package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
  160. package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
  161. package/src/acp/__tests__/session-manager.test.ts +895 -4
  162. package/src/acp/acp-claude-oauth.ts +328 -14
  163. package/src/acp/acp-credentials.ts +19 -0
  164. package/src/acp/agent-process.test.ts +4 -1
  165. package/src/acp/agent-process.ts +96 -10
  166. package/src/acp/auth-required.ts +50 -1
  167. package/src/acp/auto-install.test.ts +9 -7
  168. package/src/acp/claude-token-refresh.ts +150 -0
  169. package/src/acp/client-handler.ts +16 -3
  170. package/src/acp/model-config.test.ts +166 -0
  171. package/src/acp/model-config.ts +138 -0
  172. package/src/acp/prepare-agent-env.ts +21 -6
  173. package/src/acp/resolve-agent.test.ts +220 -6
  174. package/src/acp/resolve-agent.ts +88 -23
  175. package/src/acp/session-manager.test.ts +196 -3
  176. package/src/acp/session-manager.ts +441 -18
  177. package/src/acp/session-snapshot.ts +260 -0
  178. package/src/acp/types.ts +50 -5
  179. package/src/agent/loop.ts +20 -0
  180. package/src/api/attachment-provenance.test.ts +68 -0
  181. package/src/api/computer-use-tool.test.ts +55 -0
  182. package/src/api/computer-use-tool.ts +27 -0
  183. package/src/api/events/acp-session-model-update.test.ts +88 -0
  184. package/src/api/events/acp-session-model-update.ts +46 -0
  185. package/src/api/events/app-files-changed.ts +4 -3
  186. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  187. package/src/api/events/desktop-activity-changed.ts +10 -0
  188. package/src/api/events/notification-intent.ts +2 -0
  189. package/src/api/events/user-message-echo.ts +7 -1
  190. package/src/api/index.ts +17 -0
  191. package/src/api/responses/conversation-message.ts +8 -0
  192. package/src/approvals/approval-primitive.ts +5 -2
  193. package/src/approvals/scoped-approval-grants.ts +6 -2
  194. package/src/apps/app-store.ts +7 -6
  195. package/src/apps/resolve-app.ts +81 -0
  196. package/src/apps/source-fingerprint.ts +266 -0
  197. package/src/avatar/identity-avatar.ts +7 -2
  198. package/src/browser/operations.ts +2 -1
  199. package/src/browser/types.ts +7 -0
  200. package/src/browser/virtual-desktop-target.ts +34 -0
  201. package/src/bundler/app-compiler.ts +15 -0
  202. package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
  203. package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
  204. package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
  205. package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
  206. package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
  207. package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
  208. package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
  209. package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
  210. package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +27 -4
  211. package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
  212. package/src/calls/__tests__/voice-control-protocol.test.ts +151 -27
  213. package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
  214. package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
  215. package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
  216. package/src/calls/__tests__/voice-session-bridge.test.ts +140 -3
  217. package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
  218. package/src/calls/barge-in-guard.ts +118 -0
  219. package/src/calls/call-constants.ts +24 -4
  220. package/src/calls/call-controller.ts +499 -199
  221. package/src/calls/call-transport.ts +16 -0
  222. package/src/calls/media-stream-output.ts +32 -0
  223. package/src/calls/media-stream-server.ts +82 -30
  224. package/src/calls/media-stream-stt-session.ts +15 -0
  225. package/src/{live-voice → calls}/progress-narration.ts +17 -4
  226. package/src/calls/resolve-call-tts-provider.ts +33 -2
  227. package/src/calls/tts-call-strategy.ts +9 -5
  228. package/src/calls/types.ts +2 -0
  229. package/src/calls/voice-control-protocol.ts +194 -21
  230. package/src/calls/voice-leg-coordinator.ts +189 -0
  231. package/src/calls/voice-progress-cadence.ts +423 -0
  232. package/src/calls/voice-session-bridge.ts +140 -76
  233. package/src/calls/voice-triage-escalate.ts +178 -59
  234. package/src/cli/commands/__tests__/apps.test.ts +245 -0
  235. package/src/cli/commands/__tests__/bash.test.ts +53 -0
  236. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  237. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  238. package/src/cli/commands/__tests__/plugins.test.ts +85 -0
  239. package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
  240. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  241. package/src/cli/commands/apps.help.ts +67 -13
  242. package/src/cli/commands/apps.ts +183 -0
  243. package/src/cli/commands/bash.help.ts +9 -13
  244. package/src/cli/commands/bash.ts +7 -25
  245. package/src/cli/commands/browser.help.ts +43 -0
  246. package/src/cli/commands/browser.ts +52 -8
  247. package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
  248. package/src/cli/commands/channels/index.help.ts +70 -2
  249. package/src/cli/commands/channels/index.ts +16 -3
  250. package/src/cli/commands/channels/send.ts +128 -0
  251. package/src/cli/commands/credentials.help.ts +9 -5
  252. package/src/cli/commands/credentials.ts +2 -2
  253. package/src/cli/commands/mcp.help.ts +8 -8
  254. package/src/cli/commands/mcp.ts +1 -1
  255. package/src/cli/commands/oauth/index.help.ts +10 -0
  256. package/src/cli/commands/oauth/providers.ts +12 -0
  257. package/src/cli/commands/oauth/request.test.ts +179 -2
  258. package/src/cli/commands/oauth/request.ts +41 -10
  259. package/src/cli/commands/oauth/status.ts +5 -0
  260. package/src/cli/commands/plugins.ts +73 -23
  261. package/src/cli/commands/roadmap.help.ts +5 -1
  262. package/src/cli/commands/roadmap.ts +1 -1
  263. package/src/cli/commands/schedules.ts +2 -0
  264. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  265. package/src/cli/lib/__tests__/install-from-github.test.ts +219 -0
  266. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  267. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  268. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +251 -0
  269. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +65 -2
  270. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
  271. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
  272. package/src/cli/lib/__tests__/plugin-details.test.ts +35 -2
  273. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
  274. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
  275. package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
  276. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  277. package/src/cli/lib/bundled-marketplace.json +962 -81
  278. package/src/cli/lib/bundled-plugin-packages.json +786 -0
  279. package/src/cli/lib/bundled-plugin-packages.ts +87 -0
  280. package/src/cli/lib/diff-plugin.ts +1 -1
  281. package/src/cli/lib/inspect-plugin.ts +87 -12
  282. package/src/cli/lib/install-from-github.ts +150 -94
  283. package/src/cli/lib/install-from-platform.ts +14 -9
  284. package/src/cli/lib/list-installed-plugins.ts +21 -35
  285. package/src/cli/lib/plugin-catalog-cache.ts +22 -3
  286. package/src/cli/lib/plugin-catalog-local.ts +13 -3
  287. package/src/cli/lib/plugin-catalog-platform.ts +6 -1
  288. package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
  289. package/src/cli/lib/plugin-details.ts +33 -8
  290. package/src/cli/lib/plugin-marketplace.ts +121 -21
  291. package/src/cli/lib/plugin-pin-history.ts +5 -2
  292. package/src/cli/lib/search-plugins.ts +58 -16
  293. package/src/cli/lib/uninstall-plugin.ts +59 -1
  294. package/src/cli/lib/upgrade-plugin.ts +61 -8
  295. package/src/cli/output.ts +18 -4
  296. package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
  297. package/src/config/__tests__/default-provider.test.ts +4 -0
  298. package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
  299. package/src/config/__tests__/platform-identity.test.ts +255 -0
  300. package/src/config/__tests__/platform-rehydration.test.ts +99 -9
  301. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  302. package/src/config/acp-agent-write.ts +43 -0
  303. package/src/config/acp-defaults.test.ts +13 -3
  304. package/src/config/acp-defaults.ts +23 -6
  305. package/src/config/acp-schema.test.ts +81 -0
  306. package/src/config/acp-schema.ts +30 -2
  307. package/src/config/bundled-skills/acp/SKILL.md +23 -15
  308. package/src/config/bundled-skills/acp/TOOLS.json +7 -3
  309. package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
  310. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
  311. package/src/config/bundled-skills/computer-use/SKILL.md +59 -2
  312. package/src/config/bundled-skills/computer-use/TOOLS.json +101 -2
  313. package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
  314. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
  315. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
  316. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
  317. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  318. package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
  319. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
  320. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
  321. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
  322. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
  323. package/src/config/bundled-skills/schedule/SKILL.md +7 -8
  324. package/src/config/bundled-skills/screen-annotation/SKILL.md +10 -3
  325. package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
  326. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  327. package/src/config/bundled-skills/skill-management/TOOLS.json +13 -9
  328. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
  329. package/src/config/default-profile-catalog.ts +1 -1
  330. package/src/config/env.ts +2 -2
  331. package/src/config/feature-flag-registry.json +33 -13
  332. package/src/config/loader.ts +11 -17
  333. package/src/config/memory-v3-gate.test.ts +31 -0
  334. package/src/config/memory-v3-gate.ts +20 -16
  335. package/src/config/platform-identity.ts +201 -0
  336. package/src/config/platform-rehydration.ts +11 -27
  337. package/src/config/profile-text-generation.test.ts +51 -0
  338. package/src/config/profile-text-generation.ts +51 -0
  339. package/src/config/schema.ts +2 -2
  340. package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
  341. package/src/config/schemas/__tests__/voice.test.ts +148 -0
  342. package/src/config/schemas/calls.ts +1 -28
  343. package/src/config/schemas/live-voice.ts +1 -173
  344. package/src/config/schemas/llm.ts +7 -2
  345. package/src/config/schemas/mcp.ts +15 -9
  346. package/src/config/schemas/memory-retrospective.ts +10 -0
  347. package/src/config/schemas/monitoring.ts +9 -0
  348. package/src/config/schemas/services.ts +18 -0
  349. package/src/config/schemas/skills.ts +0 -14
  350. package/src/config/schemas/voice.ts +150 -0
  351. package/src/config/skills.ts +13 -21
  352. package/src/credential-execution/ces-runtime.ts +2 -2
  353. package/src/credential-execution/executable-discovery.ts +36 -134
  354. package/src/credential-execution/process-manager.test.ts +37 -42
  355. package/src/credential-execution/process-manager.ts +24 -19
  356. package/src/credential-health/credential-health-service.ts +11 -17
  357. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  358. package/src/daemon/__tests__/conversation-tool-setup.test.ts +109 -121
  359. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +5 -1
  360. package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
  361. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  362. package/src/daemon/app-change-notify.ts +45 -0
  363. package/src/daemon/assistant-attachments.ts +20 -12
  364. package/src/daemon/config-watcher.ts +13 -194
  365. package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
  366. package/src/daemon/conversation-agent-loop.ts +12 -0
  367. package/src/daemon/conversation-attachments.ts +93 -12
  368. package/src/daemon/conversation-client-surface.ts +26 -0
  369. package/src/daemon/conversation-error.ts +3 -2
  370. package/src/daemon/conversation-media-retry.ts +2 -1
  371. package/src/daemon/conversation-process.ts +35 -64
  372. package/src/daemon/conversation-runtime-assembly.ts +21 -8
  373. package/src/daemon/conversation-skill-tools.ts +13 -31
  374. package/src/daemon/conversation-store.ts +1 -1
  375. package/src/daemon/conversation-surfaces.ts +21 -10
  376. package/src/daemon/conversation-tool-setup.ts +135 -109
  377. package/src/daemon/conversation-turn-finalize.ts +26 -13
  378. package/src/daemon/conversation.ts +17 -3
  379. package/src/daemon/daemon-control.ts +3 -7
  380. package/src/daemon/dictation-profile-store.ts +2 -1
  381. package/src/daemon/handlers/shared.ts +2 -1
  382. package/src/daemon/handlers/skills.ts +6 -3
  383. package/src/daemon/host-bash-proxy.ts +8 -12
  384. package/src/daemon/host-browser-proxy.ts +15 -1
  385. package/src/daemon/host-cu-proxy.ts +275 -45
  386. package/src/daemon/host-file-proxy.ts +8 -10
  387. package/src/daemon/host-proxy-preactivation.ts +4 -1
  388. package/src/daemon/host-transfer-proxy.ts +14 -18
  389. package/src/daemon/lifecycle.ts +4 -9
  390. package/src/daemon/mcp-reload-service.ts +13 -6
  391. package/src/daemon/message-types/shared.ts +1 -0
  392. package/src/daemon/message-types/sync.ts +2 -0
  393. package/src/daemon/orphan-reaper.ts +4 -3
  394. package/src/daemon/persist-media-references.ts +20 -3
  395. package/src/daemon/providers-setup.ts +5 -1
  396. package/src/daemon/reaction-history-render.ts +2 -1
  397. package/src/daemon/shutdown-handlers.ts +1 -9
  398. package/src/daemon/skill-memory-refresh.ts +64 -0
  399. package/src/daemon/tool-setup-types.ts +6 -0
  400. package/src/daemon/tool-side-effects.ts +7 -18
  401. package/src/daemon/wake-conversation-ops.ts +38 -15
  402. package/src/desktop/__tests__/fake-desktop.ts +123 -0
  403. package/src/desktop/desktop-automation-lease.test.ts +351 -0
  404. package/src/desktop/desktop-automation-lease.ts +306 -0
  405. package/src/desktop/desktop-browser-client.test.ts +413 -0
  406. package/src/desktop/desktop-browser-client.ts +430 -0
  407. package/src/desktop/desktop-browser-cursor.ts +40 -0
  408. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  409. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  410. package/src/desktop/desktop-browser-operations.ts +130 -0
  411. package/src/desktop/desktop-chrome-policy.test.ts +79 -0
  412. package/src/desktop/desktop-chrome-policy.ts +47 -0
  413. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  414. package/src/desktop/desktop-chrome-session.ts +87 -0
  415. package/src/desktop/desktop-dependencies.test.ts +131 -0
  416. package/src/desktop/desktop-dependencies.ts +329 -0
  417. package/src/desktop/desktop-display.ts +9 -0
  418. package/src/desktop/desktop-panel-config.test.ts +135 -0
  419. package/src/desktop/desktop-panel-config.ts +162 -0
  420. package/src/desktop/desktop-session-manager.test.ts +813 -0
  421. package/src/desktop/desktop-session-manager.ts +970 -0
  422. package/src/desktop/desktop-stream-bridge.test.ts +395 -0
  423. package/src/desktop/desktop-stream-bridge.ts +214 -0
  424. package/src/desktop/desktop-wallpaper.test.ts +191 -0
  425. package/src/desktop/desktop-wallpaper.ts +113 -0
  426. package/src/desktop/desktop-window-drag.ts +177 -0
  427. package/src/desktop/desktop-window-manager-config.test.ts +230 -0
  428. package/src/desktop/desktop-window-manager-config.ts +238 -0
  429. package/src/desktop/desktop-window-theme.ts +170 -0
  430. package/src/desktop/virtual-desktop-feature.ts +18 -0
  431. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  432. package/src/heartbeat/heartbeat-service.ts +21 -3
  433. package/src/home/__tests__/feed-writer.test.ts +22 -0
  434. package/src/home/feed-writer.ts +11 -2
  435. package/src/hooks/types.ts +33 -0
  436. package/src/i18n/AGENTS.md +22 -0
  437. package/src/i18n/__tests__/i18n.test.ts +112 -0
  438. package/src/i18n/index.ts +16 -0
  439. package/src/i18n/locales.ts +93 -0
  440. package/src/i18n/messages.ts +70 -0
  441. package/src/i18n/title.ts +25 -0
  442. package/src/inbound/platform-callback-registration.ts +4 -5
  443. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  444. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  445. package/src/ipc/assistant-server.ts +15 -1
  446. package/src/ipc/cli-client.ts +13 -5
  447. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
  448. package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
  449. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
  450. package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
  451. package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
  452. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  453. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
  454. package/src/live-voice/__tests__/live-voice-vad.test.ts +660 -5
  455. package/src/live-voice/__tests__/protocol.test.ts +95 -0
  456. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
  457. package/src/live-voice/__tests__/session-controls.test.ts +156 -0
  458. package/src/live-voice/live-voice-metrics.ts +9 -0
  459. package/src/live-voice/live-voice-photo.ts +25 -6
  460. package/src/live-voice/live-voice-session.ts +1081 -760
  461. package/src/live-voice/protocol.ts +89 -2
  462. package/src/live-voice/session-controls.ts +197 -0
  463. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  464. package/src/mcp/__tests__/effective-config.test.ts +13 -2
  465. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  466. package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
  467. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  468. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  469. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  470. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  471. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  472. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  473. package/src/mcp/__tests__/startup.test.ts +30 -7
  474. package/src/mcp/__tests__/tool-caps.test.ts +107 -0
  475. package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
  476. package/src/mcp/client.ts +64 -16
  477. package/src/mcp/credential-target.ts +105 -0
  478. package/src/mcp/effective-config.ts +45 -18
  479. package/src/mcp/manager.ts +201 -81
  480. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  481. package/src/mcp/mcp-auth-state.ts +38 -6
  482. package/src/mcp/mcp-header-store.ts +29 -45
  483. package/src/mcp/mcp-oauth-provider.ts +101 -33
  484. package/src/mcp/spec-schema.ts +111 -0
  485. package/src/mcp/startup.ts +41 -17
  486. package/src/mcp/tool-caps.ts +176 -0
  487. package/src/mcp/workspace-mcp-config.ts +188 -0
  488. package/src/messaging/provider-message-metadata.ts +3 -3
  489. package/src/messaging/provider.ts +10 -3
  490. package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
  491. package/src/messaging/providers/callback-routing.ts +18 -0
  492. package/src/messaging/providers/channel-transport.ts +56 -0
  493. package/src/messaging/providers/discord/transport.ts +17 -0
  494. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
  495. package/src/messaging/providers/slack/adapter.ts +7 -44
  496. package/src/messaging/providers/slack/transport.ts +16 -0
  497. package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
  498. package/src/messaging/providers/telegram-bot/client.ts +0 -26
  499. package/src/messaging/providers/telegram-bot/transport.ts +20 -0
  500. package/src/messaging/providers/whatsapp/adapter.ts +1 -48
  501. package/src/messaging/providers/whatsapp/transport.ts +18 -0
  502. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  503. package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
  504. package/src/monitoring/mount-watch.ts +348 -0
  505. package/src/monitoring/plugin-auto-update.ts +6 -0
  506. package/src/monitoring/worker.ts +9 -0
  507. package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
  508. package/src/notifications/__tests__/copy-composer.test.ts +70 -0
  509. package/src/notifications/__tests__/decision-engine.test.ts +175 -0
  510. package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
  511. package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
  512. package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
  513. package/src/notifications/adapters/macos.ts +3 -0
  514. package/src/notifications/assistant-reply-producer.ts +8 -3
  515. package/src/notifications/conversation-pairing.ts +36 -16
  516. package/src/notifications/copy-composer.ts +11 -3
  517. package/src/notifications/decision-engine.ts +43 -8
  518. package/src/notifications/delivered-post-record.ts +11 -1
  519. package/src/notifications/guardian-feed-projection.ts +125 -5
  520. package/src/notifications/home-feed-side-effect.ts +55 -7
  521. package/src/notifications/schedule-result-producer.ts +64 -22
  522. package/src/oauth/AGENTS.md +4 -0
  523. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  524. package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
  525. package/src/oauth/byo-connection.test.ts +105 -0
  526. package/src/oauth/byo-connection.ts +36 -6
  527. package/src/oauth/connection-resolver.test.ts +100 -0
  528. package/src/oauth/connection-resolver.ts +120 -3
  529. package/src/oauth/identity-verifier.ts +24 -0
  530. package/src/oauth/oauth-store.ts +11 -1
  531. package/src/oauth/provider-serializer.ts +10 -0
  532. package/src/oauth/scope-utils.ts +24 -0
  533. package/src/oauth/seed-providers.ts +367 -27
  534. package/src/onboarding/checkin-event.ts +2 -1
  535. package/src/outbound-proxy/certs.ts +3 -5
  536. package/src/permissions/checker.test.ts +11 -6
  537. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  538. package/src/permissions/confirmation-guardian-request.ts +2 -3
  539. package/src/persistence/attachments-store.ts +84 -44
  540. package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
  541. package/src/persistence/conversation-crud.ts +182 -83
  542. package/src/persistence/conversation-lineage.ts +12 -4
  543. package/src/persistence/conversation-plugin-facade.ts +13 -0
  544. package/src/persistence/conversation-queries.ts +2 -2
  545. package/src/persistence/conversation-title-placeholders.ts +23 -0
  546. package/src/persistence/conversation-title-service.ts +9 -26
  547. package/src/persistence/conversation-tool-surface.ts +86 -0
  548. package/src/persistence/conversation-types.test.ts +32 -0
  549. package/src/persistence/conversation-types.ts +26 -4
  550. package/src/persistence/embeddings/qdrant-manager.ts +163 -18
  551. package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
  552. package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
  553. package/src/persistence/message-cursor.ts +78 -0
  554. package/src/persistence/message-reads.ts +30 -0
  555. package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
  556. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  557. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  558. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  559. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  560. package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
  561. package/src/persistence/schema/conversation-tool-surfaces.ts +22 -0
  562. package/src/persistence/schema/index.ts +1 -0
  563. package/src/persistence/schema/memory-core.ts +5 -0
  564. package/src/persistence/schema/oauth.ts +1 -0
  565. package/src/persistence/steps.ts +15 -3
  566. package/src/platform/client.test.ts +4 -2
  567. package/src/platform/client.ts +2 -10
  568. package/src/plugin-api/constants.ts +2 -0
  569. package/src/plugin-api/conversation-turn.ts +31 -7
  570. package/src/plugin-api/index.ts +10 -1
  571. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  572. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  573. package/src/plugin-api/types.ts +2 -0
  574. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  575. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  576. package/src/plugins/defaults/index.ts +2 -0
  577. package/src/plugins/defaults/memory/AGENTS.md +14 -2
  578. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  579. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  580. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  581. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
  582. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
  583. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
  584. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
  585. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +66 -10
  586. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +20 -0
  587. package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
  588. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
  589. package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
  590. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  591. package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
  592. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  593. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  594. package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -3
  595. package/src/plugins/defaults/memory/context-search/format.ts +4 -2
  596. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +7 -3
  597. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +4 -2
  598. package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -8
  599. package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
  600. package/src/plugins/defaults/memory/hooks/init.ts +10 -0
  601. package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
  602. package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
  603. package/src/plugins/defaults/memory/injectors.ts +4 -3
  604. package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
  605. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
  606. package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
  607. package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
  608. package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
  609. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
  610. package/src/plugins/defaults/memory/memory-retrospective-job.ts +67 -184
  611. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  612. package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
  613. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
  614. package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
  615. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  616. package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
  617. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
  618. package/src/plugins/defaults/memory/startup.ts +28 -2
  619. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  620. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  621. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  622. package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
  623. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  624. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  625. package/src/plugins/defaults/memory/substrate/page-index.ts +7 -1
  626. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
  627. package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
  628. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  629. package/src/plugins/defaults/memory/tools.ts +1 -1
  630. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
  631. package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -1
  632. package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
  633. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  634. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  635. package/src/plugins/defaults/memory/v2/reranker.ts +7 -1
  636. package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
  637. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  638. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  639. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
  640. package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
  641. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +51 -0
  642. package/src/plugins/defaults/memory/v3/card.ts +4 -2
  643. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  644. package/src/plugins/defaults/memory/v3/orchestrate.ts +86 -22
  645. package/src/plugins/defaults/memory/v3/pool-select.ts +13 -8
  646. package/src/plugins/defaults/memory/v3/sections.ts +10 -3
  647. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +13 -1
  648. package/src/plugins/defaults/memory/worker.ts +6 -0
  649. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  650. package/src/plugins/external-plugin-loader.ts +49 -86
  651. package/src/plugins/installed-plugin-dirs.ts +6 -4
  652. package/src/plugins/mcp-servers.ts +24 -90
  653. package/src/plugins/mtime-cache.ts +2 -1
  654. package/src/plugins/user-loader.ts +2 -1
  655. package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
  656. package/src/providers/__tests__/retry-callsite.test.ts +41 -0
  657. package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
  658. package/src/providers/anthropic/client.ts +0 -47
  659. package/src/providers/gemini/client.ts +9 -23
  660. package/src/providers/gemini-thought-signature.test.ts +63 -0
  661. package/src/providers/gemini-thought-signature.ts +44 -0
  662. package/src/providers/inference/__tests__/endpoint-probe.test.ts +87 -3
  663. package/src/providers/inference/adapter-factory.ts +6 -0
  664. package/src/providers/inference/auth.ts +2 -3
  665. package/src/providers/inference/connection-availability.ts +27 -8
  666. package/src/providers/inference/endpoint-probe.ts +36 -12
  667. package/src/providers/inference/resolve-auth.ts +22 -4
  668. package/src/providers/inference/service-account-token.ts +248 -0
  669. package/src/providers/jev/client.test.ts +260 -0
  670. package/src/providers/jev/client.ts +518 -0
  671. package/src/providers/model-catalog.ts +92 -3
  672. package/src/providers/model-intents.ts +10 -5
  673. package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
  674. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
  675. package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
  676. package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
  677. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
  678. package/src/providers/openai/chat-completions-provider.ts +212 -79
  679. package/src/providers/openai/codex-models.ts +0 -4
  680. package/src/providers/openai/google-thought-signature.ts +320 -0
  681. package/src/providers/openai/responses-provider.ts +9 -2
  682. package/src/providers/opencode/client.test.ts +240 -4
  683. package/src/providers/opencode/client.ts +129 -3
  684. package/src/providers/openrouter/client.ts +20 -0
  685. package/src/providers/outbound-request-sanitize.ts +96 -0
  686. package/src/providers/retry.ts +30 -19
  687. package/src/providers/unparseable-tool-args.ts +5 -1
  688. package/src/runtime/AGENTS.md +9 -3
  689. package/src/runtime/__tests__/agent-wake.test.ts +86 -1
  690. package/src/runtime/__tests__/channel-send.test.ts +443 -0
  691. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +132 -0
  692. package/src/runtime/__tests__/host-observe.test.ts +40 -1
  693. package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
  694. package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
  695. package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
  696. package/src/runtime/agent-wake.ts +22 -5
  697. package/src/runtime/assistant-event-hub.ts +21 -3
  698. package/src/runtime/auth/same-actor.ts +38 -3
  699. package/src/runtime/channel-send.ts +316 -0
  700. package/src/runtime/guardian-action-service.ts +2 -17
  701. package/src/runtime/guardian-reply-router.ts +1 -8
  702. package/src/runtime/host-observe.ts +11 -1
  703. package/src/runtime/http-server.ts +160 -151
  704. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  705. package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
  706. package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
  707. package/src/runtime/routes/__tests__/acp-routes.test.ts +178 -4
  708. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
  709. package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
  710. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
  711. package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
  712. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
  713. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  714. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
  715. package/src/runtime/routes/__tests__/plugins-routes.test.ts +127 -3
  716. package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
  717. package/src/runtime/routes/acp-routes.test.ts +15 -0
  718. package/src/runtime/routes/acp-routes.ts +84 -165
  719. package/src/runtime/routes/app-management-routes.ts +90 -2
  720. package/src/runtime/routes/app-routes.ts +1 -1
  721. package/src/runtime/routes/browser-context.ts +39 -0
  722. package/src/runtime/routes/browser-routes.ts +16 -47
  723. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  724. package/src/runtime/routes/channel-route-shared.ts +1 -9
  725. package/src/runtime/routes/channel-send-routes.ts +85 -0
  726. package/src/runtime/routes/conversation-list-routes.ts +14 -3
  727. package/src/runtime/routes/conversation-query-routes.ts +40 -0
  728. package/src/runtime/routes/conversation-routes.ts +45 -7
  729. package/src/runtime/routes/credential-routes.ts +20 -14
  730. package/src/runtime/routes/desktop-setup-routes.test.ts +89 -0
  731. package/src/runtime/routes/desktop-setup-routes.ts +41 -0
  732. package/src/runtime/routes/diagnostics-routes.ts +61 -4
  733. package/src/runtime/routes/events-routes.ts +4 -0
  734. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  735. package/src/runtime/routes/host-cu-routes.ts +6 -0
  736. package/src/runtime/routes/inbound-message-handler.ts +2 -3
  737. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
  738. package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
  739. package/src/runtime/routes/index.ts +4 -2
  740. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  741. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  742. package/src/runtime/routes/log-export-routes.ts +2 -0
  743. package/src/runtime/routes/mcp-auth-routes.ts +141 -137
  744. package/src/runtime/routes/migration-routes.ts +6 -20
  745. package/src/runtime/routes/oauth-commands-routes.ts +98 -45
  746. package/src/runtime/routes/oauth-providers.ts +11 -0
  747. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  748. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  749. package/src/runtime/routes/platform-routes.ts +18 -15
  750. package/src/runtime/routes/plugins-routes.ts +83 -50
  751. package/src/runtime/routes/schedule-routes.ts +2 -0
  752. package/src/runtime/routes/secret-routes.ts +99 -119
  753. package/src/runtime/routes/workspace-routes.test.ts +55 -26
  754. package/src/runtime/routes/workspace-routes.ts +46 -63
  755. package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
  756. package/src/runtime/routes/workspace-tree-walk.ts +240 -0
  757. package/src/runtime/routes/workspace-utils.ts +17 -0
  758. package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
  759. package/src/runtime/services/conversation-serializer.ts +16 -3
  760. package/src/runtime/sync/resource-sync-events.ts +6 -1
  761. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  762. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  763. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  764. package/src/schedule/run-script.ts +4 -3
  765. package/src/schedule/schedule-store.ts +9 -2
  766. package/src/schedule/scheduler.ts +47 -11
  767. package/src/schedule/tool-surface-readiness.ts +65 -0
  768. package/src/schedule/worker-mcp.ts +85 -0
  769. package/src/schedule/worker.ts +9 -31
  770. package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
  771. package/src/security/ces-http-record-backend.ts +120 -0
  772. package/src/security/ces-rpc-record-backend.ts +7 -0
  773. package/src/security/secure-keys.ts +60 -48
  774. package/src/security/untrusted-content.ts +3 -1
  775. package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
  776. package/src/skills/managed-store.ts +153 -31
  777. package/src/skills/workspace-skill-md-signature.ts +50 -0
  778. package/src/subagent/consult-context.ts +12 -25
  779. package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
  780. package/src/tools/acp/list-agents.test.ts +3 -1
  781. package/src/tools/acp/spawn.test.ts +146 -11
  782. package/src/tools/acp/spawn.ts +21 -3
  783. package/src/tools/acp/status.test.ts +349 -0
  784. package/src/tools/acp/status.ts +112 -9
  785. package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
  786. package/src/tools/browser/browser-execution.ts +45 -14
  787. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  788. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
  789. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  790. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
  791. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  792. package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
  793. package/src/tools/browser/network-recorder.ts +2 -1
  794. package/src/tools/capability-offer.test.ts +22 -2
  795. package/src/tools/capability-offer.ts +16 -1
  796. package/src/tools/computer-use/definitions.ts +123 -5
  797. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
  798. package/src/tools/credentials/metadata-store.ts +100 -0
  799. package/src/tools/filesystem/search.ts +2 -1
  800. package/src/tools/host-filesystem/edit.test.ts +2 -1
  801. package/src/tools/host-filesystem/edit.ts +2 -2
  802. package/src/tools/host-filesystem/read.test.ts +2 -1
  803. package/src/tools/host-filesystem/read.ts +2 -2
  804. package/src/tools/host-filesystem/transfer.test.ts +2 -1
  805. package/src/tools/host-filesystem/transfer.ts +2 -2
  806. package/src/tools/host-filesystem/write.test.ts +2 -1
  807. package/src/tools/host-filesystem/write.ts +2 -2
  808. package/src/tools/host-terminal/host-shell.test.ts +16 -0
  809. package/src/tools/host-terminal/host-shell.ts +16 -26
  810. package/src/tools/network/web-search.ts +6 -1
  811. package/src/tools/policy-context.ts +6 -7
  812. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  813. package/src/tools/shared/shell-output.test.ts +10 -0
  814. package/src/tools/shared/shell-output.ts +14 -2
  815. package/src/tools/skills/find-similar-skills.test.ts +214 -2
  816. package/src/tools/skills/find-similar-skills.ts +62 -9
  817. package/src/tools/skills/resolve-execute-invocation.ts +24 -0
  818. package/src/tools/skills/sandbox-runner.ts +13 -2
  819. package/src/tools/skills/scaffold-managed.ts +78 -16
  820. package/src/tools/subagent/spawn.ts +5 -8
  821. package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
  822. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +43 -0
  823. package/src/tools/terminal/safe-env.ts +35 -24
  824. package/src/tools/terminal/sanitized-bash.ts +118 -0
  825. package/src/tools/terminal/shell-launch.test.ts +162 -0
  826. package/src/tools/terminal/shell.test.ts +29 -0
  827. package/src/tools/terminal/shell.ts +13 -7
  828. package/src/tools/ui-surface/channel-variants.ts +10 -59
  829. package/src/tools/watch/watch-retro-report.ts +0 -8
  830. package/src/types/host-capabilities.ts +5 -3
  831. package/src/util/__tests__/language-subtag.test.ts +23 -1
  832. package/src/util/abort-reasons.ts +12 -1
  833. package/src/util/ca-bundle.ts +13 -0
  834. package/src/util/host-process.test.ts +17 -1
  835. package/src/util/host-process.ts +24 -0
  836. package/src/util/language-subtag.ts +16 -0
  837. package/src/util/platform.ts +8 -0
  838. package/src/util/plugin-manifest.ts +203 -0
  839. package/src/util/unicode.ts +44 -1
  840. package/src/watch/watch-retro.ts +6 -8
  841. package/src/watch/watch-timeline.ts +2 -2
  842. package/src/workflows/run-manager.ts +2 -1
  843. package/src/workspace/git-porcelain.ts +66 -0
  844. package/src/workspace/git-service.ts +6 -55
  845. package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
  846. package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
  847. package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
  848. package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
  849. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
  850. package/src/workspace/migrations/registry.ts +6 -0
  851. package/src/__tests__/app-source-watcher.test.ts +0 -218
  852. package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
  853. package/src/__tests__/speaker-identification.test.ts +0 -56
  854. package/src/calls/speaker-identification.ts +0 -233
  855. package/src/daemon/app-source-watcher.ts +0 -218
  856. package/src/messaging/providers/whatsapp/client.ts +0 -29
  857. package/src/runtime/routes/debug-bash-routes.ts +0 -173
  858. package/src/util/fs-watcher-error.ts +0 -36
  859. /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
@@ -42,9 +42,14 @@ const MAX_HISTORY_ENTRIES = 10;
42
42
  const LOOP_DETECTION_WINDOW = 3;
43
43
  const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
44
44
 
45
+ const SEQUENCE_TOOL = "computer_use_sequence";
46
+
47
+ const SCREENSHOT_OMITTED_MESSAGE =
48
+ "Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
49
+
45
50
  // computer_use_key combos that change only selection/cursor/clipboard state.
46
- // The AX tree models none of these, so they always produce an empty diff —
47
- // exempt them from the "NO VISIBLE EFFECT" signal (mirrors computer_use_wait).
51
+ // The AX tree models none of these, so they always produce an empty diff and
52
+ // are exempt from the unchanged-tree signal.
48
53
  // Stored in canonical form (see canonicalizeKeyCombo): modifier aliases
49
54
  // normalized and ordered, so `cmd + a`, `command+a`, `alt+tab`, `tab+shift`
50
55
  // all match.
@@ -113,6 +118,8 @@ export interface CuObservationResult {
113
118
  executionResult?: string;
114
119
  executionError?: string;
115
120
  userGuidance?: string;
121
+ /** Per-phase helper timings in milliseconds, absent on older helpers. */
122
+ timings?: Record<string, number>;
116
123
  }
117
124
 
118
125
  export interface ActionRecord {
@@ -122,12 +129,28 @@ export interface ActionRecord {
122
129
  reasoning?: string;
123
130
  }
124
131
 
132
+ // Steps whose empty diff says nothing about whether they worked: waiting and
133
+ // observing change nothing by design, and an AppleScript reports its own
134
+ // outcome in the execution result, often for an app whose content the tree
135
+ // cannot see. They neither warn nor move the unchanged streak.
136
+ const NO_AX_DIFF_TOOLS = new Set([
137
+ "computer_use_wait",
138
+ "computer_use_observe",
139
+ "computer_use_run_applescript",
140
+ ]);
141
+
142
+ type ActionIdentity = Pick<ActionRecord, "toolName" | "input">;
143
+
144
+ function isNoDiffTool(action: ActionIdentity | undefined): boolean {
145
+ return action !== undefined && NO_AX_DIFF_TOOLS.has(action.toolName);
146
+ }
147
+
125
148
  /**
126
149
  * True when `action` is a computer_use_key press whose key only mutates
127
- * selection/cursor/clipboard state — changes the AX tree cannot represent, so
150
+ * selection/cursor/clipboard state: changes the AX tree cannot represent, so
128
151
  * an empty diff is expected rather than a sign the action did nothing.
129
152
  */
130
- function isNoDiffKeyAction(action: ActionRecord | undefined): boolean {
153
+ function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
131
154
  if (action?.toolName !== "computer_use_key") {
132
155
  return false;
133
156
  }
@@ -161,6 +184,40 @@ function actionSignature(record: ActionRecord): string {
161
184
  return `${record.toolName}:${JSON.stringify(record.input)}`;
162
185
  }
163
186
 
187
+ /**
188
+ * The key a desktop's first look is tracked under. An untargeted request has
189
+ * no client id, so it gets a key no real id can take.
190
+ */
191
+ function observedTargetKey(targetClientId: string | undefined): string {
192
+ return targetClientId === undefined
193
+ ? "\u0000untargeted"
194
+ : `client:${targetClientId}`;
195
+ }
196
+
197
+ /** Whether `input` scopes the observation to one window or display. */
198
+ function hasCaptureTarget(input: Record<string, unknown>): boolean {
199
+ return (
200
+ Object.hasOwn(input, "capture_window_id") ||
201
+ Object.hasOwn(input, "captureWindowId") ||
202
+ Object.hasOwn(input, "captureDisplayId")
203
+ );
204
+ }
205
+
206
+ /**
207
+ * Whether the resolved client claimed `capability` on its connection. An
208
+ * untargeted request has no client to vouch for it, so it never qualifies.
209
+ */
210
+ function clientAdvertises(
211
+ targetClientId: string | undefined,
212
+ capability: "host_cu_window_capture" | "host_cu_sequence",
213
+ ): boolean {
214
+ const client =
215
+ targetClientId == null
216
+ ? undefined
217
+ : assistantEventHub.getClientById(targetClientId);
218
+ return client?.capabilities.includes(capability) ?? false;
219
+ }
220
+
164
221
  // ---------------------------------------------------------------------------
165
222
  // HostCuProxy
166
223
  // ---------------------------------------------------------------------------
@@ -172,8 +229,42 @@ export class HostCuProxy {
172
229
  private _previousAXTree: string | undefined;
173
230
  private _consecutiveUnchangedSteps = 0;
174
231
  private _actionHistory: ActionRecord[] = [];
175
- /** Owned request IDs mapped to whether their observation is scoped. */
176
- private _ownedRequests = new Map<string, boolean>();
232
+ /**
233
+ * Desktops an unscoped observation has come back from since the last reset,
234
+ * keyed by `observedTargetKey`. A first look is per desktop: having seen one
235
+ * machine says nothing about another the conversation switches to.
236
+ */
237
+ private _observedTargets = new Set<string>();
238
+ /**
239
+ * Bumped on every reset. A request carries the value it was dispatched
240
+ * under, so an observation that lands after a reset cannot restore state
241
+ * that reset cleared.
242
+ */
243
+ private _resetGeneration = 0;
244
+ /**
245
+ * Desktops a request has been sent to since the last reset, keyed by
246
+ * `observedTargetKey`, so the end of a task can reach each one.
247
+ */
248
+ private _dispatchedTargets = new Map<string, string | undefined>();
249
+ /**
250
+ * Owned request IDs mapped to whether their observation is scoped, whether
251
+ * the helper was told to skip the screenshot, and when the request was
252
+ * dispatched. The dispatch time gives the round trip, which is the part of
253
+ * a step the helper's own timings cannot see.
254
+ */
255
+ private _ownedRequests = new Map<
256
+ string,
257
+ {
258
+ scoped: boolean;
259
+ screenshotSkipped: boolean;
260
+ targetKey: string;
261
+ resetGeneration: number;
262
+ dispatchedAt: number;
263
+ toolName: string;
264
+ input: Record<string, unknown>;
265
+ step: number;
266
+ }
267
+ >();
177
268
 
178
269
  constructor(maxSteps = loadConfig().maxStepsPerSession) {
179
270
  this._maxSteps = maxSteps;
@@ -288,11 +379,7 @@ export class HostCuProxy {
288
379
  }
289
380
  // Never dispatch this option to a legacy executor: it would forward the
290
381
  // unknown snake-case key and silently capture the entire desktop.
291
- const client =
292
- resolvedTargetClientId == null
293
- ? undefined
294
- : assistantEventHub.getClientById(resolvedTargetClientId);
295
- if (!client?.capabilities.includes("host_cu_window_capture")) {
382
+ if (!clientAdvertises(resolvedTargetClientId, "host_cu_window_capture")) {
296
383
  return Promise.resolve({
297
384
  content:
298
385
  "Window-only observation requires a connected client advertising host_cu_window_capture; update the desktop app before retrying. No capture was requested.",
@@ -300,14 +387,40 @@ export class HostCuProxy {
300
387
  });
301
388
  }
302
389
  }
303
- const scopedObservation =
304
- hasWindowTarget ||
305
- Object.hasOwn(input, "captureWindowId") ||
306
- Object.hasOwn(input, "captureDisplayId");
390
+ // An older helper answers an unknown tool name as done, which would end
391
+ // the session with nothing run.
392
+ if (
393
+ toolName === SEQUENCE_TOOL &&
394
+ !clientAdvertises(resolvedTargetClientId, "host_cu_sequence")
395
+ ) {
396
+ return Promise.resolve({
397
+ content:
398
+ "Batched actions require a desktop app that supports computer_use_sequence; update the desktop app, or send the actions one at a time. Nothing was run.",
399
+ isError: true,
400
+ });
401
+ }
402
+ const scopedObservation = hasCaptureTarget(input);
307
403
  if (scopedObservation) {
308
404
  this._previousAXTree = undefined;
309
405
  this._consecutiveUnchangedSteps = 0;
310
406
  }
407
+ const targetKey = observedTargetKey(resolvedTargetClientId);
408
+ // Pointing never reaches the helper's capture path, so its input goes out
409
+ // as given. Every other request carries the screenshot decision made now,
410
+ // at dispatch, and never the model-facing snake_case key.
411
+ let dispatchInput = input;
412
+ let screenshotSkipped = false;
413
+ if (toolName !== POINT_AT_PROXY_TOOL) {
414
+ const { include_screenshot: _includeScreenshot, ...rest } = input;
415
+ screenshotSkipped = !this.shouldAttachScreenshot(
416
+ toolName,
417
+ input,
418
+ targetKey,
419
+ );
420
+ dispatchInput = screenshotSkipped
421
+ ? { ...rest, includeScreenshot: false }
422
+ : rest;
423
+ }
311
424
  const requestId = uuid();
312
425
 
313
426
  return new Promise<ToolExecutionResult>((resolve, reject) => {
@@ -351,7 +464,17 @@ export class HostCuProxy {
351
464
  detachAbort = () => signal.removeEventListener("abort", onAbort);
352
465
  }
353
466
 
354
- this._ownedRequests.set(requestId, scopedObservation);
467
+ this._dispatchedTargets.set(targetKey, resolvedTargetClientId);
468
+ this._ownedRequests.set(requestId, {
469
+ scoped: scopedObservation,
470
+ screenshotSkipped,
471
+ targetKey,
472
+ resetGeneration: this._resetGeneration,
473
+ dispatchedAt: Date.now(),
474
+ toolName,
475
+ input,
476
+ step: stepNumber,
477
+ });
355
478
 
356
479
  pendingInteractions.register(requestId, {
357
480
  conversationId,
@@ -375,7 +498,7 @@ export class HostCuProxy {
375
498
  requestId,
376
499
  conversationId,
377
500
  toolName,
378
- input,
501
+ input: dispatchInput,
379
502
  stepNumber,
380
503
  reasoning,
381
504
  ...(resolvedTargetClientId != null
@@ -403,7 +526,8 @@ export class HostCuProxy {
403
526
  requestId: string,
404
527
  observation: CuObservationResult,
405
528
  ): ToolExecutionResult | undefined {
406
- const scopedObservation = this._ownedRequests.get(requestId) ?? false;
529
+ const owned = this._ownedRequests.get(requestId);
530
+ const scopedObservation = owned?.scoped ?? false;
407
531
  this._ownedRequests.delete(requestId);
408
532
  const interaction = pendingInteractions.resolve(requestId, "answered");
409
533
  if (!interaction?.rpcResolve) {
@@ -411,20 +535,64 @@ export class HostCuProxy {
411
535
  return undefined;
412
536
  }
413
537
 
538
+ // Label the line from what this request was dispatched with, never from
539
+ // current proxy state. One model response can dispatch several CU tools,
540
+ // and the agent loop runs them concurrently, so a later call can advance
541
+ // the history and the step count before an earlier observation lands.
542
+ // Reading them here would file each measurement under whichever tool was
543
+ // dispatched last, and would mislabel computer_use_point_at, which never
544
+ // records an action at all.
545
+ if (owned) {
546
+ log.info(
547
+ {
548
+ requestId,
549
+ toolName: owned.toolName,
550
+ step: owned.step,
551
+ roundTripMs: Date.now() - owned.dispatchedAt,
552
+ ...(observation.timings ? { helper: observation.timings } : {}),
553
+ },
554
+ "Host CU step timings",
555
+ );
556
+ }
557
+
414
558
  // A targeted snapshot has no comparable action/diff baseline; neither it
415
- // nor the first desktop observation after it can imply "no visible effect".
416
- if (scopedObservation) {
559
+ // nor the first desktop observation after it can imply an unchanged tree.
560
+ // A response dispatched before the last reset belongs to a finished run,
561
+ // so it must not change any state the new run has started building.
562
+ const fromCurrentRun =
563
+ owned === undefined || owned.resetGeneration === this._resetGeneration;
564
+ if (scopedObservation && fromCurrentRun) {
417
565
  this._previousAXTree = undefined;
418
566
  this._consecutiveUnchangedSteps = 0;
419
567
  }
420
568
  const prevAXTree = this._previousAXTree;
569
+ // Judge the empty diff against the action this request carried, for the
570
+ // same reason the timings line does.
571
+ const action: ActionIdentity | undefined = owned
572
+ ? { toolName: owned.toolName, input: owned.input }
573
+ : this.lastRecordedAction();
421
574
  const comparableObservation = scopedObservation
422
575
  ? { ...observation, axDiff: undefined, secondaryWindows: undefined }
423
576
  : observation;
424
- if (!scopedObservation) {
425
- this.updateStateFromObservation(comparableObservation);
577
+ // Pointing observes nothing, so its response leaves the screen state alone.
578
+ if (
579
+ fromCurrentRun &&
580
+ !scopedObservation &&
581
+ owned?.toolName !== POINT_AT_PROXY_TOOL
582
+ ) {
583
+ this.updateStateFromObservation(comparableObservation, action);
584
+ // A desktop has had its first look only once pixels from it arrived,
585
+ // so a failed capture leaves the next request asking again.
586
+ if (owned && observation.screenshot) {
587
+ this._observedTargets.add(owned.targetKey);
588
+ }
426
589
  }
427
- const result = this.formatObservation(comparableObservation, prevAXTree);
590
+ const result = this.formatObservation(
591
+ comparableObservation,
592
+ prevAXTree,
593
+ owned?.screenshotSkipped ?? false,
594
+ action,
595
+ );
428
596
  interaction.rpcResolve(result);
429
597
  return result;
430
598
  }
@@ -454,12 +622,65 @@ export class HostCuProxy {
454
622
  }
455
623
  }
456
624
 
625
+ /**
626
+ * Finish the task: tell every desktop this task drove that it is over, then
627
+ * reset. The notice rides `host_cu_cancel` with a fresh request ID, which
628
+ * matches nothing in flight; the helper treats any cancel as the run
629
+ * stopping and puts the pointer back where the user left it, instead of
630
+ * waiting out its idle fallback.
631
+ */
632
+ endTask(conversationId: string): void {
633
+ for (const targetClientId of this._dispatchedTargets.values()) {
634
+ try {
635
+ broadcastMessage(
636
+ {
637
+ type: "host_cu_cancel",
638
+ requestId: uuid(),
639
+ conversationId,
640
+ ...(targetClientId != null ? { targetClientId } : {}),
641
+ },
642
+ conversationId,
643
+ { targetClientId },
644
+ );
645
+ } catch {
646
+ // Best-effort: the helper still returns the pointer after its idle delay.
647
+ }
648
+ }
649
+ this.reset();
650
+ }
651
+
457
652
  /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
458
653
  reset(): void {
654
+ this._dispatchedTargets.clear();
459
655
  this._stepCount = 0;
460
656
  this._previousAXTree = undefined;
461
657
  this._consecutiveUnchangedSteps = 0;
462
658
  this._actionHistory = [];
659
+ this._observedTargets.clear();
660
+ this._resetGeneration++;
661
+ }
662
+
663
+ /**
664
+ * Whether the request about to be dispatched should carry a screenshot. The
665
+ * accessibility tree comes back every step. Pixels come back after every
666
+ * action, so the model sees what its action did, on the first look since
667
+ * the last reset, for a window- or display-scoped capture, and when the
668
+ * model asks for them. Only a plain observation leaves whether the tree is
669
+ * enough to the model.
670
+ */
671
+ private shouldAttachScreenshot(
672
+ toolName: string,
673
+ input: Record<string, unknown>,
674
+ targetKey: string,
675
+ ): boolean {
676
+ if (toolName !== "computer_use_observe") {
677
+ return true;
678
+ }
679
+ return (
680
+ !this._observedTargets.has(targetKey) ||
681
+ hasCaptureTarget(input) ||
682
+ input.include_screenshot === true
683
+ );
463
684
  }
464
685
 
465
686
  // ---------------------------------------------------------------------------
@@ -474,6 +695,8 @@ export class HostCuProxy {
474
695
  formatObservation(
475
696
  obs: CuObservationResult,
476
697
  previousAXTree?: string,
698
+ screenshotSkipped = false,
699
+ action: ActionIdentity | undefined = this.lastRecordedAction(),
477
700
  ): ToolExecutionResult {
478
701
  const prevTree = previousAXTree;
479
702
  const parts: string[] = [];
@@ -492,24 +715,17 @@ export class HostCuProxy {
492
715
  parts.push(obs.axDiff);
493
716
  parts.push("");
494
717
  } else if (prevTree != null && obs.axTree != null) {
495
- const lastAction =
496
- this._actionHistory.length > 0
497
- ? this._actionHistory[this._actionHistory.length - 1]
498
- : undefined;
499
- const isWaitAction = lastAction?.toolName === "computer_use_wait";
500
- const isNoDiffKey = isNoDiffKeyAction(lastAction);
501
-
502
- if (!isWaitAction && !isNoDiffKey) {
718
+ if (!isNoDiffTool(action) && !isNoDiffKeyAction(action)) {
503
719
  if (
504
720
  this._consecutiveUnchangedSteps >=
505
721
  CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
506
722
  ) {
507
723
  parts.push(
508
- `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
724
+ `WARNING: the accessibility tree did not change across ${this._consecutiveUnchangedSteps} consecutive actions. If the screenshot does not show them working either, try a different approach.`,
509
725
  );
510
726
  } else {
511
727
  parts.push(
512
- "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
728
+ "The accessibility tree did not change after your last action. Apps that draw their own content (timelines, canvases, spreadsheet grids) change without it, so check the screenshot before deciding the action did nothing.",
513
729
  );
514
730
  }
515
731
  parts.push("");
@@ -552,6 +768,15 @@ export class HostCuProxy {
552
768
  parts.push(...screenshotMeta);
553
769
  }
554
770
 
771
+ // Only a deliberate omission is announced, and only beside a tree the line
772
+ // can point at. A step that asked for pixels and got none is a capture
773
+ // failure, which this line must not disguise. A refused or failed step
774
+ // still carries the line, since that is where asking for pixels helps most.
775
+ const isError = obs.executionError != null;
776
+ if (screenshotSkipped && !obs.screenshot && obs.axTree) {
777
+ parts.push("", SCREENSHOT_OMITTED_MESSAGE);
778
+ }
779
+
555
780
  const content = parts.join("\n").trim() || "Action executed";
556
781
 
557
782
  const contentBlocks: ContentBlock[] = [];
@@ -566,8 +791,6 @@ export class HostCuProxy {
566
791
  });
567
792
  }
568
793
 
569
- const isError = obs.executionError != null;
570
-
571
794
  return {
572
795
  content: isError
573
796
  ? `Action failed: ${obs.executionError}\n\n${content}`
@@ -617,19 +840,26 @@ export class HostCuProxy {
617
840
  // Private helpers
618
841
  // ---------------------------------------------------------------------------
619
842
 
620
- private updateStateFromObservation(obs: CuObservationResult): void {
843
+ private lastRecordedAction(): ActionRecord | undefined {
844
+ return this._actionHistory.at(-1);
845
+ }
846
+
847
+ private updateStateFromObservation(
848
+ obs: CuObservationResult,
849
+ action: ActionIdentity | undefined,
850
+ ): void {
621
851
  if (this._stepCount > 0) {
622
- const lastAction =
623
- this._actionHistory.length > 0
624
- ? this._actionHistory[this._actionHistory.length - 1]
625
- : undefined;
626
- if (obs.axDiff != null || isNoDiffKeyAction(lastAction)) {
852
+ if (obs.axDiff != null || isNoDiffKeyAction(action)) {
627
853
  // A real diff, or an exempt key whose effect is invisible by design,
628
- // breaks the no-effect streak — clear it rather than preserving a
629
- // stale count so an intervening cmd+a can't bridge two no-op actions
630
- // into a false "consecutive" escalation.
854
+ // breaks the no-effect streak. Clearing it rather than preserving a
855
+ // stale count keeps an intervening cmd+a from bridging two no-op
856
+ // actions into a false "consecutive" escalation.
631
857
  this._consecutiveUnchangedSteps = 0;
632
- } else if (this._previousAXTree != null && obs.axTree != null) {
858
+ } else if (
859
+ !isNoDiffTool(action) &&
860
+ this._previousAXTree != null &&
861
+ obs.axTree != null
862
+ ) {
633
863
  this._consecutiveUnchangedSteps++;
634
864
  }
635
865
  }
@@ -6,10 +6,10 @@ import {
6
6
  broadcastMessage,
7
7
  } from "../runtime/assistant-event-hub.js";
8
8
  import {
9
- ambiguousSameUserError,
10
9
  enforceSameActorOrErrorResult,
11
- pickSameUserAutoResolve,
10
+ pickMostRecentSameUserClient,
12
11
  snapshotHostProxyActorPrincipalId,
12
+ unavailableSameUserClientError,
13
13
  } from "../runtime/auth/same-actor.js";
14
14
  import * as pendingInteractions from "../runtime/pending-interactions.js";
15
15
  import { readAudioBase64 } from "../tools/shared/filesystem/audio-read.js";
@@ -102,19 +102,17 @@ export class HostFileProxy {
102
102
  });
103
103
  }
104
104
  } else {
105
- // Auto-resolve to the unique same-user client. Reject ambiguous
106
- // (multi-machine) cases so a single targeted-style request cannot
107
- // fan out across the user's machines.
108
- const resolved = pickSameUserAutoResolve({
105
+ resolvedTargetClientId = pickMostRecentSameUserClient({
109
106
  hub: assistantEventHub,
110
107
  capability: "host_file",
111
108
  sourceActorPrincipalId,
112
109
  });
113
- if (resolved.kind === "ambiguous") {
114
- return Promise.resolve(ambiguousSameUserError("host_file"));
110
+ if (
111
+ resolvedTargetClientId === undefined &&
112
+ sourceActorPrincipalId !== undefined
113
+ ) {
114
+ return Promise.resolve(unavailableSameUserClientError("host_file"));
115
115
  }
116
- resolvedTargetClientId =
117
- resolved.kind === "match" ? resolved.clientId : undefined;
118
116
  }
119
117
 
120
118
  // Same-user check: targeted host_file requests must be bound to the same
@@ -9,6 +9,8 @@
9
9
  * - `daemon/process-message.ts` (create path, prepareConversationForMessage)
10
10
  * - `daemon/conversation-process.ts` `drainSingleMessage` (re-add after dequeue)
11
11
  * - `daemon/conversation-process.ts` `drainBatch` (re-add after dequeue)
12
+ * - `calls/voice-session-bridge.ts` (tool-capable legs of a live-voice
13
+ * session opened from the macOS desktop client)
12
14
  *
13
15
  * The create paths additionally instantiate the proxy itself; that
14
16
  * instantiation logic is per-proxy-class and stays inline at each create
@@ -105,12 +107,13 @@ export const HOST_PROXY_SKILL_PREACTIVATIONS: ReadonlyArray<{
105
107
  */
106
108
  /**
107
109
  * Capabilities a client asks for on its connection rather than getting from
108
- * what it is. Both ride the host_cu transport and are answered only by a
110
+ * what it is. Each rides the host_cu transport and is answered only by a
109
111
  * macOS build new enough to send the header (`events-routes.ts`), so the
110
112
  * interface alone cannot say whether one is really there.
111
113
  */
112
114
  const NEGOTIATED_CAPABILITIES: ReadonlySet<HostProxyCapability> = new Set([
113
115
  "host_cu_window_capture",
116
+ "host_cu_sequence",
114
117
  "host_cu_annotate",
115
118
  ]);
116
119
 
@@ -10,10 +10,10 @@ import {
10
10
  broadcastMessage,
11
11
  } from "../runtime/assistant-event-hub.js";
12
12
  import {
13
- ambiguousSameUserError,
14
13
  enforceSameActorOrErrorResult,
15
- pickSameUserAutoResolve,
14
+ pickMostRecentSameUserClient,
16
15
  snapshotHostProxyActorPrincipalId,
16
+ unavailableSameUserClientError,
17
17
  } from "../runtime/auth/same-actor.js";
18
18
  import * as pendingInteractions from "../runtime/pending-interactions.js";
19
19
  import type { ToolExecutionResult } from "../tools/types.js";
@@ -162,19 +162,17 @@ export class HostTransferProxy {
162
162
  });
163
163
  }
164
164
  } else {
165
- // Auto-resolve to the unique same-user client; reject ambiguous
166
- // (multi-machine) cases so a single targeted-style transfer cannot
167
- // fan out across the user's machines.
168
- const resolved = pickSameUserAutoResolve({
165
+ resolvedTargetClientId = pickMostRecentSameUserClient({
169
166
  hub: assistantEventHub,
170
167
  capability: "host_file",
171
168
  sourceActorPrincipalId,
172
169
  });
173
- if (resolved.kind === "ambiguous") {
174
- return Promise.resolve(ambiguousSameUserError("host_file"));
170
+ if (
171
+ resolvedTargetClientId === undefined &&
172
+ sourceActorPrincipalId !== undefined
173
+ ) {
174
+ return Promise.resolve(unavailableSameUserClientError("host_file"));
175
175
  }
176
- resolvedTargetClientId =
177
- resolved.kind === "match" ? resolved.clientId : undefined;
178
176
  }
179
177
 
180
178
  if (resolvedTargetClientId != null) {
@@ -364,19 +362,17 @@ export class HostTransferProxy {
364
362
  });
365
363
  }
366
364
  } else {
367
- // Auto-resolve to the unique same-user client; reject ambiguous
368
- // (multi-machine) cases so a single targeted-style transfer cannot
369
- // fan out across the user's machines.
370
- const resolved = pickSameUserAutoResolve({
365
+ resolvedTargetClientId = pickMostRecentSameUserClient({
371
366
  hub: assistantEventHub,
372
367
  capability: "host_file",
373
368
  sourceActorPrincipalId,
374
369
  });
375
- if (resolved.kind === "ambiguous") {
376
- return Promise.resolve(ambiguousSameUserError("host_file"));
370
+ if (
371
+ resolvedTargetClientId === undefined &&
372
+ sourceActorPrincipalId !== undefined
373
+ ) {
374
+ return Promise.resolve(unavailableSameUserClientError("host_file"));
377
375
  }
378
- resolvedTargetClientId =
379
- resolved.kind === "match" ? resolved.clientId : undefined;
380
376
  }
381
377
 
382
378
  if (resolvedTargetClientId != null) {
@@ -71,7 +71,6 @@ import { ensureCompleteCustomProfiles } from "../workspace/custom-profile-ensure
71
71
  import { ensureDefaultProvider } from "../workspace/default-provider-ensure.js";
72
72
  import { WORKSPACE_MIGRATIONS } from "../workspace/migrations/registry.js";
73
73
  import { runWorkspaceMigrations } from "../workspace/migrations/runner.js";
74
- import { startAppSourceWatcher } from "./app-source-watcher.js";
75
74
  import { startConfigWatcher } from "./config-watcher.js";
76
75
  import { startConversationEvictor } from "./conversation-evictor.js";
77
76
  import { writePid } from "./daemon-control.js";
@@ -674,11 +673,11 @@ export async function runDaemon(): Promise<void> {
674
673
  // blocked.
675
674
  startConsentRefresh();
676
675
 
677
- // Bring up the daemon's CES connection (process + handshake + reconnect
676
+ // Bring up the assistant's CES connection (process + handshake + reconnect
678
677
  // wiring). Blocks up to a 20s timeout so credential reads route through CES
679
- // before provider init; non-fatal — falls back to the direct credential store
680
- // on failure. The sidecar accepts exactly one bootstrap connection, so this
681
- // happens at the process level.
678
+ // before provider init; non-fatal, falls back to the direct credential store
679
+ // on failure. CES serves a multi-connection bootstrap socket, so this
680
+ // happens at the process level and child processes can connect independently.
682
681
  await startCes(config);
683
682
 
684
683
  // Bring up the plugin layer: install the runtime bridge, register the
@@ -706,10 +705,6 @@ export async function runDaemon(): Promise<void> {
706
705
  // config, and broadcast the relevant resource-changed events to clients.
707
706
  startConfigWatcher();
708
707
 
709
- // Watch app source directories so edits recompile + refresh surfaces across
710
- // all conversations.
711
- startAppSourceWatcher();
712
-
713
708
  // Start the CLI IPC server. Throws on EADDRINUSE to abort startup when another
714
709
  // daemon already holds the socket, so this process never runs background jobs
715
710
  // against the shared database as an unmanageable duplicate.