@vellumai/assistant 0.4.48 → 0.4.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (252) hide show
  1. package/ARCHITECTURE.md +2 -2
  2. package/README.md +2 -23
  3. package/docs/architecture/integrations.md +45 -41
  4. package/docs/architecture/keychain-broker.md +3 -3
  5. package/docs/runbook-trusted-contacts.md +3 -8
  6. package/hook-templates/debug-prompt-logger/hook.json +1 -1
  7. package/hook-templates/debug-prompt-logger/run.sh +1 -3
  8. package/package.json +1 -1
  9. package/src/__tests__/actor-token-service.test.ts +0 -1
  10. package/src/__tests__/anthropic-provider.test.ts +156 -0
  11. package/src/__tests__/approval-cascade.test.ts +810 -0
  12. package/src/__tests__/approval-primitive.test.ts +0 -1
  13. package/src/__tests__/approval-routes-http.test.ts +2 -0
  14. package/src/__tests__/assistant-attachments.test.ts +12 -34
  15. package/src/__tests__/assistant-feature-flag-guardrails.test.ts +76 -0
  16. package/src/__tests__/assistant-feature-flags-integration.test.ts +0 -1
  17. package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
  18. package/src/__tests__/channel-guardian.test.ts +0 -2
  19. package/src/__tests__/channel-readiness-routes.test.ts +15 -6
  20. package/src/__tests__/channel-readiness-service.test.ts +10 -9
  21. package/src/__tests__/checker.test.ts +9 -29
  22. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  23. package/src/__tests__/computer-use-tools.test.ts +2 -19
  24. package/src/__tests__/config-watcher.test.ts +0 -1
  25. package/src/__tests__/confirmation-request-guardian-bridge.test.ts +0 -1
  26. package/src/__tests__/context-image-dimensions.test.ts +332 -0
  27. package/src/__tests__/context-token-estimator.test.ts +196 -13
  28. package/src/__tests__/conversation-attention-store.test.ts +0 -1
  29. package/src/__tests__/conversation-attention-telegram.test.ts +0 -1
  30. package/src/__tests__/conversation-routes-guardian-reply.test.ts +144 -0
  31. package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
  32. package/src/__tests__/credential-metadata-store.test.ts +64 -73
  33. package/src/__tests__/credential-security-invariants.test.ts +13 -7
  34. package/src/__tests__/credential-vault-unit.test.ts +280 -49
  35. package/src/__tests__/credential-vault.test.ts +138 -16
  36. package/src/__tests__/credentials-cli.test.ts +71 -0
  37. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +0 -1
  38. package/src/__tests__/ephemeral-permissions.test.ts +3 -3
  39. package/src/__tests__/gateway-only-guard.test.ts +0 -1
  40. package/src/__tests__/guardian-action-grant-mint-consume.test.ts +0 -1
  41. package/src/__tests__/guardian-decision-primitive-canonical.test.ts +0 -1
  42. package/src/__tests__/guardian-routing-invariants.test.ts +0 -1
  43. package/src/__tests__/guardian-verification-voice-binding.test.ts +0 -1
  44. package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -39
  45. package/src/__tests__/heartbeat-service.test.ts +0 -1
  46. package/src/__tests__/host-cu-proxy.test.ts +629 -0
  47. package/src/__tests__/host-shell-tool.test.ts +27 -15
  48. package/src/__tests__/http-user-message-parity.test.ts +1 -0
  49. package/src/__tests__/ingress-url-consistency.test.ts +14 -21
  50. package/src/__tests__/integration-status.test.ts +32 -51
  51. package/src/__tests__/intent-routing.test.ts +0 -1
  52. package/src/__tests__/invite-routes-http.test.ts +10 -9
  53. package/src/__tests__/keychain-broker-client.test.ts +11 -43
  54. package/src/__tests__/notification-routing-intent.test.ts +0 -1
  55. package/src/__tests__/oauth-cli.test.ts +373 -14
  56. package/src/__tests__/oauth-provider-profiles.test.ts +9 -9
  57. package/src/__tests__/oauth-scope-policy.test.ts +4 -6
  58. package/src/__tests__/oauth-store.test.ts +756 -0
  59. package/src/__tests__/onboarding-starter-tasks.test.ts +0 -1
  60. package/src/__tests__/provider-error-scenarios.test.ts +0 -1
  61. package/src/__tests__/provider-streaming.benchmark.test.ts +0 -1
  62. package/src/__tests__/public-ingress-urls.test.ts +15 -21
  63. package/src/__tests__/recording-handler.test.ts +3 -4
  64. package/src/__tests__/registry.test.ts +2 -2
  65. package/src/__tests__/runtime-events-sse.test.ts +55 -7
  66. package/src/__tests__/schedule-store.test.ts +0 -1
  67. package/src/__tests__/scheduler-recurrence.test.ts +0 -1
  68. package/src/__tests__/scoped-approval-grants.test.ts +0 -1
  69. package/src/__tests__/scoped-grant-security-matrix.test.ts +0 -1
  70. package/src/__tests__/secret-ingress-handler.test.ts +0 -1
  71. package/src/__tests__/send-endpoint-busy.test.ts +21 -6
  72. package/src/__tests__/sequence-store.test.ts +0 -1
  73. package/src/__tests__/session-init.benchmark.test.ts +4 -5
  74. package/src/__tests__/skill-include-graph.test.ts +66 -0
  75. package/src/__tests__/skill-load-feature-flag.test.ts +0 -1
  76. package/src/__tests__/skill-load-tool.test.ts +149 -1
  77. package/src/__tests__/skill-projection-feature-flag.test.ts +0 -1
  78. package/src/__tests__/skills-uninstall.test.ts +1 -1
  79. package/src/__tests__/skills.test.ts +3 -3
  80. package/src/__tests__/slack-channel-config.test.ts +67 -3
  81. package/src/__tests__/slack-share-routes.test.ts +17 -19
  82. package/src/__tests__/system-prompt.test.ts +0 -1
  83. package/src/__tests__/telegram-invite-adapter.test.ts +18 -22
  84. package/src/__tests__/terminal-tools.test.ts +4 -3
  85. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  86. package/src/__tests__/tool-approval-handler.test.ts +0 -1
  87. package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +0 -1
  88. package/src/__tests__/tool-executor-lifecycle-events.test.ts +0 -1
  89. package/src/__tests__/tool-executor-shell-integration.test.ts +0 -1
  90. package/src/__tests__/tool-executor.test.ts +0 -1
  91. package/src/__tests__/tool-grant-request-escalation.test.ts +0 -1
  92. package/src/__tests__/trust-store-pattern-matches.test.ts +29 -0
  93. package/src/__tests__/trust-store.test.ts +1 -22
  94. package/src/__tests__/trusted-contact-approval-notifier.test.ts +0 -1
  95. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +0 -1
  96. package/src/__tests__/twilio-routes.test.ts +0 -16
  97. package/src/__tests__/verification-control-plane-policy.test.ts +0 -1
  98. package/src/__tests__/voice-scoped-grant-consumer.test.ts +0 -1
  99. package/src/agent/ax-tree-compaction.test.ts +235 -0
  100. package/src/agent/loop.ts +76 -130
  101. package/src/calls/call-domain.ts +1 -6
  102. package/src/calls/relay-server.ts +9 -13
  103. package/src/calls/twilio-config.ts +2 -7
  104. package/src/calls/twilio-routes.ts +1 -2
  105. package/src/calls/voice-ingress-preflight.ts +1 -1
  106. package/src/cli/commands/browser-relay.ts +18 -12
  107. package/src/cli/commands/completions.ts +0 -3
  108. package/src/cli/commands/credentials.ts +101 -15
  109. package/src/cli/commands/oauth/apps.ts +255 -0
  110. package/src/cli/commands/oauth/connections.ts +299 -0
  111. package/src/cli/commands/oauth/index.ts +52 -0
  112. package/src/cli/commands/oauth/providers.ts +242 -0
  113. package/src/cli/commands/skills.ts +4 -338
  114. package/src/cli/program.ts +1 -5
  115. package/src/cli/reference.ts +1 -3
  116. package/src/config/assistant-feature-flags.ts +0 -3
  117. package/src/config/bundled-skills/_shared/CLI_RETRIEVAL_PATTERN.md +1 -1
  118. package/src/config/bundled-skills/computer-use/SKILL.md +3 -6
  119. package/src/config/bundled-skills/computer-use/TOOLS.json +22 -4
  120. package/src/config/bundled-skills/google-calendar/calendar-client.ts +21 -16
  121. package/src/config/bundled-skills/messaging/tools/shared.ts +1 -4
  122. package/src/config/bundled-skills/settings/SKILL.md +1 -1
  123. package/src/config/bundled-skills/settings/TOOLS.json +2 -8
  124. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +5 -33
  125. package/src/config/env-registry.ts +14 -83
  126. package/src/config/env.ts +11 -50
  127. package/src/config/feature-flag-registry.json +16 -16
  128. package/src/config/loader.ts +0 -6
  129. package/src/config/schema.ts +3 -1
  130. package/src/config/skills.ts +21 -2
  131. package/src/context/image-dimensions.ts +229 -0
  132. package/src/context/token-estimator.ts +75 -12
  133. package/src/context/window-manager.ts +49 -10
  134. package/src/daemon/assistant-attachments.ts +1 -13
  135. package/src/daemon/handlers/config-ingress.ts +8 -33
  136. package/src/daemon/handlers/config-slack-channel.ts +49 -46
  137. package/src/daemon/handlers/config-telegram.ts +32 -16
  138. package/src/daemon/handlers/sessions.ts +10 -24
  139. package/src/daemon/handlers/shared.ts +0 -130
  140. package/src/daemon/host-cu-proxy.ts +401 -0
  141. package/src/daemon/lifecycle.ts +36 -68
  142. package/src/daemon/message-protocol.ts +3 -0
  143. package/src/daemon/message-types/computer-use.ts +2 -119
  144. package/src/daemon/message-types/host-cu.ts +19 -0
  145. package/src/daemon/message-types/messages.ts +3 -0
  146. package/src/daemon/server.ts +14 -21
  147. package/src/daemon/session-agent-loop-handlers.ts +2 -0
  148. package/src/daemon/session-attachments.ts +1 -2
  149. package/src/daemon/session-slash.ts +1 -1
  150. package/src/daemon/session-surfaces.ts +40 -28
  151. package/src/daemon/session-tool-setup.ts +2 -9
  152. package/src/daemon/session.ts +138 -15
  153. package/src/daemon/tool-side-effects.ts +2 -8
  154. package/src/daemon/watch-handler.ts +2 -2
  155. package/src/events/tool-metrics-listener.ts +2 -2
  156. package/src/hooks/manager.ts +1 -4
  157. package/src/inbound/public-ingress-urls.ts +7 -7
  158. package/src/logfire.ts +16 -5
  159. package/src/memory/conversation-key-store.ts +21 -0
  160. package/src/memory/db-init.ts +4 -0
  161. package/src/memory/migrations/149-oauth-tables.ts +60 -0
  162. package/src/memory/migrations/index.ts +1 -0
  163. package/src/memory/schema/index.ts +1 -0
  164. package/src/memory/schema/oauth.ts +65 -0
  165. package/src/messaging/provider.ts +4 -4
  166. package/src/messaging/providers/gmail/client.ts +82 -2
  167. package/src/messaging/providers/gmail/people-client.ts +10 -10
  168. package/src/messaging/providers/telegram-bot/adapter.ts +17 -17
  169. package/src/messaging/providers/whatsapp/adapter.ts +11 -8
  170. package/src/messaging/registry.ts +2 -32
  171. package/src/notifications/copy-composer.ts +0 -5
  172. package/src/notifications/signal.ts +4 -5
  173. package/src/oauth/byo-connection.test.ts +126 -25
  174. package/src/oauth/byo-connection.ts +22 -6
  175. package/src/oauth/connect-orchestrator.ts +113 -57
  176. package/src/oauth/connect-types.ts +17 -23
  177. package/src/oauth/connection-resolver.ts +35 -11
  178. package/src/oauth/connection.ts +1 -1
  179. package/src/oauth/manual-token-connection.ts +104 -0
  180. package/src/oauth/oauth-store.ts +496 -0
  181. package/src/oauth/platform-connection.test.ts +29 -0
  182. package/src/oauth/platform-connection.ts +6 -5
  183. package/src/oauth/provider-behaviors.ts +124 -0
  184. package/src/oauth/scope-policy.ts +9 -2
  185. package/src/oauth/seed-providers.ts +161 -0
  186. package/src/oauth/token-persistence.ts +74 -78
  187. package/src/permissions/checker.ts +3 -3
  188. package/src/permissions/defaults.ts +0 -1
  189. package/src/permissions/prompter.ts +10 -1
  190. package/src/permissions/trust-store.ts +13 -0
  191. package/src/prompts/__tests__/build-cli-reference-section.test.ts +3 -1
  192. package/src/prompts/system-prompt.ts +28 -40
  193. package/src/providers/anthropic/client.ts +133 -24
  194. package/src/providers/retry.ts +1 -27
  195. package/src/runtime/auth/route-policy.ts +0 -3
  196. package/src/runtime/channel-reply-delivery.ts +0 -40
  197. package/src/runtime/gateway-client.ts +0 -7
  198. package/src/runtime/http-server.ts +8 -6
  199. package/src/runtime/http-types.ts +2 -2
  200. package/src/runtime/middleware/twilio-validation.ts +1 -11
  201. package/src/runtime/pending-interactions.ts +14 -12
  202. package/src/runtime/routes/channel-delivery-routes.ts +0 -1
  203. package/src/runtime/routes/conversation-routes.ts +73 -19
  204. package/src/runtime/routes/events-routes.ts +21 -11
  205. package/src/runtime/routes/host-cu-routes.ts +97 -0
  206. package/src/runtime/routes/inbound-stages/background-dispatch.ts +12 -111
  207. package/src/runtime/routes/integrations/slack/share.ts +6 -7
  208. package/src/runtime/routes/log-export-routes.ts +126 -8
  209. package/src/runtime/routes/settings-routes.ts +55 -48
  210. package/src/runtime/routes/surface-action-routes.ts +1 -1
  211. package/src/runtime/routes/watch-routes.ts +128 -0
  212. package/src/schedule/integration-status.ts +10 -9
  213. package/src/security/credential-key.ts +0 -156
  214. package/src/security/keychain-broker-client.ts +5 -6
  215. package/src/security/oauth2.ts +1 -1
  216. package/src/security/token-manager.ts +119 -46
  217. package/src/skills/catalog-install.ts +358 -0
  218. package/src/skills/include-graph.ts +32 -0
  219. package/src/telegram/bot-username.ts +2 -3
  220. package/src/tools/browser/network-recorder.ts +1 -1
  221. package/src/tools/browser/network-recording-types.ts +1 -1
  222. package/src/tools/computer-use/definitions.ts +46 -11
  223. package/src/tools/computer-use/registry.ts +4 -5
  224. package/src/tools/credentials/broker.ts +1 -2
  225. package/src/tools/credentials/metadata-store.ts +17 -121
  226. package/src/tools/credentials/vault.ts +94 -167
  227. package/src/tools/registry.ts +2 -7
  228. package/src/tools/skills/load.ts +62 -3
  229. package/src/tools/watch/watch-state.ts +0 -12
  230. package/src/util/logger.ts +7 -41
  231. package/src/util/platform.ts +9 -28
  232. package/src/watcher/providers/google-calendar.ts +2 -1
  233. package/src/__tests__/computer-use-session-compaction.test.ts +0 -143
  234. package/src/__tests__/computer-use-session-lifecycle.test.ts +0 -322
  235. package/src/__tests__/computer-use-session-working-dir.test.ts +0 -166
  236. package/src/__tests__/computer-use-skill-baseline.test.ts +0 -78
  237. package/src/__tests__/computer-use-skill-endstate.test.ts +0 -105
  238. package/src/__tests__/computer-use-skill-lifecycle-cleanup.test.ts +0 -249
  239. package/src/__tests__/ride-shotgun-handler.test.ts +0 -452
  240. package/src/cli/commands/dev.ts +0 -129
  241. package/src/cli/commands/map.ts +0 -391
  242. package/src/cli/commands/oauth.ts +0 -77
  243. package/src/config/bundled-skills/computer-use/tools/computer-use-request-control.ts +0 -16
  244. package/src/daemon/computer-use-session.ts +0 -1026
  245. package/src/daemon/ride-shotgun-handler.ts +0 -569
  246. package/src/oauth/provider-base-urls.ts +0 -21
  247. package/src/oauth/provider-profiles.ts +0 -192
  248. package/src/prompts/computer-use-prompt.ts +0 -98
  249. package/src/runtime/routes/computer-use-routes.ts +0 -641
  250. package/src/runtime/telegram-streaming-delivery.test.ts +0 -729
  251. package/src/runtime/telegram-streaming-delivery.ts +0 -393
  252. package/src/tools/computer-use/request-computer-control.ts +0 -56
@@ -15,6 +15,7 @@ import {
15
15
  getQdrantUrlEnv,
16
16
  getRuntimeHttpHost,
17
17
  getRuntimeHttpPort,
18
+ setIngressPublicBaseUrl,
18
19
  validateEnv,
19
20
  } from "../config/env.js";
20
21
  import { loadConfig } from "../config/loader.js";
@@ -22,7 +23,7 @@ import { HeartbeatService } from "../heartbeat/heartbeat-service.js";
22
23
  import { getHookManager } from "../hooks/manager.js";
23
24
  import { installTemplates } from "../hooks/templates.js";
24
25
  import { closeSentry, initSentry } from "../instrument.js";
25
- import { initLogfire } from "../logfire.js";
26
+ import { disableLogfire, initLogfire } from "../logfire.js";
26
27
  import { getMcpServerManager } from "../mcp/manager.js";
27
28
  import * as attachmentsStore from "../memory/attachments-store.js";
28
29
  import {
@@ -40,6 +41,8 @@ import {
40
41
  emitNotificationSignal,
41
42
  registerBroadcastFn,
42
43
  } from "../notifications/emit-signal.js";
44
+ import { backfillManualTokenConnections } from "../oauth/manual-token-connection.js";
45
+ import { seedOAuthProviders } from "../oauth/seed-providers.js";
43
46
  import { ensurePromptFiles } from "../prompts/system-prompt.js";
44
47
  import { syncUpdateBulletinOnStartup } from "../prompts/update-bulletin.js";
45
48
  import { buildAssistantEvent } from "../runtime/assistant-event.js";
@@ -54,8 +57,6 @@ import {
54
57
  import { ensureVellumGuardianBinding } from "../runtime/guardian-vellum-migration.js";
55
58
  import { RuntimeHttpServer } from "../runtime/http-server.js";
56
59
  import { startScheduler } from "../schedule/scheduler.js";
57
- import { migrateKeys } from "../security/credential-key.js";
58
- import { watchSessions } from "../tools/watch/watch-state.js";
59
60
  import { getLogger, initLogger } from "../util/logger.js";
60
61
  import {
61
62
  ensureDataDir,
@@ -95,10 +96,6 @@ import {
95
96
  registerMessagingProviders,
96
97
  registerWatcherProviders,
97
98
  } from "./providers-setup.js";
98
- import {
99
- handleRideShotgunStart,
100
- handleRideShotgunStop,
101
- } from "./ride-shotgun-handler.js";
102
99
  import { seedInterfaceFiles } from "./seed-files.js";
103
100
  import { DaemonServer } from "./server.js";
104
101
  import { initSlashPairingContext } from "./session-slash.js";
@@ -137,11 +134,6 @@ export async function runDaemon(): Promise<void> {
137
134
 
138
135
  ensureDataDir();
139
136
 
140
- // Migrate legacy colon-delimited credential keys to the new
141
- // slash-delimited format. Must run after ensureDataDir() so the
142
- // secure key store is available, and before any credential reads.
143
- migrateKeys();
144
-
145
137
  // Load (or generate + persist) the auth signing key so tokens survive
146
138
  // daemon restarts. Must happen after ensureDataDir() creates the
147
139
  // protected directory.
@@ -165,6 +157,12 @@ export async function runDaemon(): Promise<void> {
165
157
  );
166
158
  }
167
159
  initializeDb();
160
+ // Seed well-known OAuth provider configurations (insert-if-not-exists)
161
+ seedOAuthProviders();
162
+ // Backfill oauth_connection rows for manual-token providers (Telegram,
163
+ // Slack channel) that already have keychain credentials from before the
164
+ // oauth_connection migration. Safe to call on every startup.
165
+ await backfillManualTokenConnections();
168
166
  log.info("Daemon startup: DB initialized");
169
167
 
170
168
  // Ensure a vellum guardian binding exists and mint the CLI edge token
@@ -242,6 +240,19 @@ export async function runDaemon(): Promise<void> {
242
240
  log.info("Daemon startup: loading config");
243
241
  const config = loadConfig();
244
242
 
243
+ // Seed module-level ingress state from the workspace config so that
244
+ // getIngressPublicBaseUrl() returns the correct value immediately after
245
+ // startup (before any handleIngressConfig("set") call). Without this,
246
+ // code paths that read the module-level state directly (e.g. session-slash
247
+ // pairing info) would see undefined until an explicit set.
248
+ if (config.ingress.enabled && config.ingress.publicBaseUrl) {
249
+ setIngressPublicBaseUrl(config.ingress.publicBaseUrl);
250
+ log.info(
251
+ { url: config.ingress.publicBaseUrl },
252
+ "Daemon startup: seeded ingress URL from workspace config",
253
+ );
254
+ }
255
+
245
256
  if (config.logFile.dir) {
246
257
  initLogger({
247
258
  dir: config.logFile.dir,
@@ -259,6 +270,18 @@ export async function runDaemon(): Promise<void> {
259
270
  await closeSentry();
260
271
  }
261
272
 
273
+ // If Logfire observability is not explicitly enabled, disable it so
274
+ // wrapWithLogfire() calls during provider setup become no-ops. Logfire
275
+ // is initialized eagerly (before config loads) for the same reason as
276
+ // Sentry — but the feature flag gates whether it actually traces.
277
+ const logfireEnabled = isAssistantFeatureFlagEnabled(
278
+ "feature_flags.logfire.enabled",
279
+ config,
280
+ );
281
+ if (!logfireEnabled) {
282
+ disableLogfire();
283
+ }
284
+
262
285
  await initializeProvidersAndTools(config);
263
286
 
264
287
  // Start the DaemonServer (session manager) before Qdrant so HTTP
@@ -501,64 +524,9 @@ export async function runDaemon(): Promise<void> {
501
524
  );
502
525
  },
503
526
  },
504
- getComputerUseDeps: () => {
527
+ getWatchDeps: () => {
505
528
  const ctx = server.getHandlerContext();
506
529
  return {
507
- cuSessions: ctx.cuSessions,
508
- sharedRequestTimestamps: ctx.sharedRequestTimestamps,
509
- cuObservationParseSequence: ctx.cuObservationParseSequence,
510
- handleRideShotgunStart: async (params) => {
511
- // The handler generates its own watchId/sessionId and
512
- // sends them via ctx.send as a watch_started message.
513
- // We intercept send to capture the IDs before they broadcast.
514
- let capturedWatchId = "";
515
- let capturedSessionId = "";
516
- const interceptCtx = {
517
- ...ctx,
518
- send: (msg: ServerMessage) => {
519
- if (
520
- "type" in msg &&
521
- msg.type === "watch_started" &&
522
- "watchId" in msg &&
523
- "sessionId" in msg
524
- ) {
525
- capturedWatchId = (msg as { watchId: string }).watchId;
526
- capturedSessionId = (msg as { sessionId: string }).sessionId;
527
- }
528
- ctx.send(msg);
529
- },
530
- };
531
- await handleRideShotgunStart(
532
- {
533
- type: "ride_shotgun_start",
534
- durationSeconds: params.durationSeconds,
535
- intervalSeconds: params.intervalSeconds,
536
- mode: params.mode,
537
- targetDomain: params.targetDomain,
538
- navigateDomain: params.navigateDomain,
539
- autoNavigate: params.autoNavigate,
540
- },
541
- interceptCtx,
542
- );
543
- return { watchId: capturedWatchId, sessionId: capturedSessionId };
544
- },
545
- handleRideShotgunStop: async (watchId) => {
546
- await handleRideShotgunStop(
547
- { type: "ride_shotgun_stop", watchId },
548
- ctx,
549
- );
550
- },
551
- getRideShotgunStatus: (watchId) => {
552
- const session = watchSessions.get(watchId);
553
- if (!session) return undefined;
554
- return {
555
- status: session.status,
556
- sessionId: session.sessionId,
557
- recordingId: session.recordingId,
558
- savedRecordingPath: session.savedRecordingPath,
559
- bootstrapFailureReason: session.bootstrapFailureReason,
560
- };
561
- },
562
530
  handleWatchObservation: async (params) => {
563
531
  await handleWatchObservation(
564
532
  {
@@ -21,6 +21,7 @@ export * from "./message-types/diagnostics.js";
21
21
  export * from "./message-types/documents.js";
22
22
  export * from "./message-types/guardian-actions.js";
23
23
  export * from "./message-types/host-bash.js";
24
+ export * from "./message-types/host-cu.js";
24
25
  export * from "./message-types/host-file.js";
25
26
  export * from "./message-types/inbox.js";
26
27
  export * from "./message-types/integrations.js";
@@ -69,6 +70,7 @@ import type {
69
70
  _GuardianActionsServerMessages,
70
71
  } from "./message-types/guardian-actions.js";
71
72
  import type { _HostBashServerMessages } from "./message-types/host-bash.js";
73
+ import type { _HostCuServerMessages } from "./message-types/host-cu.js";
72
74
  import type { _HostFileServerMessages } from "./message-types/host-file.js";
73
75
  import type {
74
76
  _InboxClientMessages,
@@ -180,6 +182,7 @@ export type ServerMessage =
180
182
  | _DocumentsServerMessages
181
183
  | _GuardianActionsServerMessages
182
184
  | _HostBashServerMessages
185
+ | _HostCuServerMessages
183
186
  | _HostFileServerMessages
184
187
  | _MemoryServerMessages
185
188
  | _WorkspaceServerMessages
@@ -1,49 +1,9 @@
1
- // Computer use, task routing, ride shotgun, and watch observation types.
1
+ // Computer use, task routing, and watch observation types.
2
2
 
3
3
  import type { CommandIntent, UserMessageAttachment } from "./shared.js";
4
4
 
5
5
  // === Client → Server ===
6
6
 
7
- export interface CuSessionCreate {
8
- type: "cu_session_create";
9
- sessionId: string;
10
- task: string;
11
- screenWidth: number;
12
- screenHeight: number;
13
- attachments?: UserMessageAttachment[];
14
- interactionType?: "computer_use" | "text_qa";
15
- }
16
-
17
- export interface CuSessionAbort {
18
- type: "cu_session_abort";
19
- sessionId: string;
20
- }
21
-
22
- export interface CuObservation {
23
- type: "cu_observation";
24
- sessionId: string;
25
- axTree?: string;
26
- axDiff?: string;
27
- secondaryWindows?: string;
28
- screenshot?: string;
29
- /** Screenshot image width in pixels (`Px`). */
30
- screenshotWidthPx?: number;
31
- /** Screenshot image height in pixels (`Px`). */
32
- screenshotHeightPx?: number;
33
- /** Screen width in macOS points (`Pt`) used by native execution. */
34
- screenWidthPt?: number;
35
- /** Screen height in macOS points (`Pt`) used by native execution. */
36
- screenHeightPt?: number;
37
- /** Coordinate origin convention used by the observation payload. */
38
- coordinateOrigin?: "top_left";
39
- /** Display ID used by screenshot capture for this observation. */
40
- captureDisplayId?: number;
41
- executionResult?: string;
42
- executionError?: string;
43
- /** Free-form guidance from the user, injected mid-turn to steer the agent. */
44
- userGuidance?: string;
45
- }
46
-
47
7
  export interface TaskSubmit {
48
8
  type: "task_submit";
49
9
  task: string;
@@ -55,22 +15,6 @@ export interface TaskSubmit {
55
15
  commandIntent?: CommandIntent;
56
16
  }
57
17
 
58
- export interface RideShotgunStart {
59
- type: "ride_shotgun_start";
60
- durationSeconds: number;
61
- intervalSeconds: number;
62
- mode?: "observe" | "learn";
63
- targetDomain?: string;
64
- /** Domain to auto-navigate (may differ from targetDomain, e.g. open.spotify.com vs spotify.com). */
65
- navigateDomain?: string;
66
- autoNavigate?: boolean;
67
- }
68
-
69
- export interface RideShotgunStop {
70
- type: "ride_shotgun_stop";
71
- watchId: string;
72
- }
73
-
74
18
  export interface WatchObservation {
75
19
  type: "watch_observation";
76
20
  watchId: string;
@@ -145,58 +89,16 @@ export interface RecordingResume {
145
89
  recordingId: string;
146
90
  }
147
91
 
148
- export interface CuAction {
149
- type: "cu_action";
150
- sessionId: string;
151
- toolName: string;
152
- input: Record<string, unknown>;
153
- reasoning?: string;
154
- stepNumber: number;
155
- }
156
-
157
- export interface CuComplete {
158
- type: "cu_complete";
159
- sessionId: string;
160
- summary: string;
161
- stepCount: number;
162
- isResponse?: boolean;
163
- }
164
-
165
- export interface CuError {
166
- type: "cu_error";
167
- sessionId: string;
168
- message: string;
169
- }
170
-
171
92
  export interface TaskRouted {
172
93
  type: "task_routed";
173
94
  sessionId: string;
174
95
  interactionType: "computer_use" | "text_qa";
175
96
  /** The task text passed to the escalated session. */
176
97
  task?: string;
177
- /** Set when a text_qa session escalates to computer_use via computer_use_request_control. */
98
+ /** Set when a text_qa session escalates to computer_use. */
178
99
  escalatedFrom?: string;
179
100
  }
180
101
 
181
- export interface RideShotgunProgress {
182
- type: "ride_shotgun_progress";
183
- watchId: string;
184
- message: string;
185
- networkEntryCount?: number;
186
- statusMessage?: string;
187
- idleHint?: boolean;
188
- }
189
-
190
- export interface RideShotgunResult {
191
- type: "ride_shotgun_result";
192
- sessionId: string;
193
- watchId: string;
194
- summary: string;
195
- observationCount: number;
196
- recordingId?: string;
197
- recordingPath?: string;
198
- }
199
-
200
102
  export interface WatchStarted {
201
103
  type: "watch_started";
202
104
  sessionId: string;
@@ -211,34 +113,15 @@ export interface WatchCompleteRequest {
211
113
  watchId: string;
212
114
  }
213
115
 
214
- /** Server → Client: bootstrap failure during learn-mode recording setup. */
215
- export interface RideShotgunError {
216
- type: "ride_shotgun_error";
217
- watchId: string;
218
- sessionId: string;
219
- message: string;
220
- }
221
-
222
116
  // --- Domain-level union aliases (consumed by the barrel file) ---
223
117
 
224
118
  export type _ComputerUseClientMessages =
225
- | CuSessionCreate
226
- | CuSessionAbort
227
- | CuObservation
228
119
  | TaskSubmit
229
- | RideShotgunStart
230
- | RideShotgunStop
231
120
  | WatchObservation
232
121
  | RecordingStatus;
233
122
 
234
123
  export type _ComputerUseServerMessages =
235
- | CuAction
236
- | CuComplete
237
- | CuError
238
124
  | TaskRouted
239
- | RideShotgunProgress
240
- | RideShotgunResult
241
- | RideShotgunError
242
125
  | WatchStarted
243
126
  | WatchCompleteRequest
244
127
  | RecordingStart
@@ -0,0 +1,19 @@
1
+ // Host computer-use proxy types.
2
+ // Enables proxying computer-use actions (click, type, screenshot, etc.)
3
+ // to the desktop client when running as a managed assistant.
4
+
5
+ // === Server → Client ===
6
+
7
+ export interface HostCuRequest {
8
+ type: "host_cu_request";
9
+ requestId: string;
10
+ sessionId: string;
11
+ toolName: string; // "computer_use_click", "computer_use_type_text", etc.
12
+ input: Record<string, unknown>;
13
+ stepNumber: number;
14
+ reasoning?: string;
15
+ }
16
+
17
+ // --- Domain-level union aliases (consumed by the barrel file) ---
18
+
19
+ export type _HostCuServerMessages = HostCuRequest;
@@ -88,6 +88,7 @@ export interface ToolOutputChunk {
88
88
  type: "tool_output_chunk";
89
89
  chunk: string;
90
90
  sessionId?: string;
91
+ toolUseId?: string;
91
92
  subType?: "tool_start" | "tool_complete" | "status";
92
93
  subToolName?: string;
93
94
  subToolInput?: string;
@@ -155,6 +156,8 @@ export interface ConfirmationRequest {
155
156
  persistentDecisionsAllowed?: boolean;
156
157
  /** Which temporary approval options the client should render (e.g. "Allow for 10 minutes", "Allow for this thread"). */
157
158
  temporaryOptionsAvailable?: Array<"allow_10m" | "allow_thread">;
159
+ /** The tool_use block ID for client-side correlation with specific tool calls. */
160
+ toolUseId?: string;
158
161
  }
159
162
 
160
163
  export interface SecretRequest {
@@ -48,7 +48,6 @@ import {
48
48
  getWorkspacePromptPath,
49
49
  } from "../util/platform.js";
50
50
  import { registerDaemonCallbacks } from "../work-items/work-item-runner.js";
51
- import { ComputerUseSession } from "./computer-use-session.js";
52
51
  import { ConfigWatcher } from "./config-watcher.js";
53
52
  import { parseIdentityFields } from "./handlers/identity.js";
54
53
  import type {
@@ -57,6 +56,7 @@ import type {
57
56
  } from "./handlers/shared.js";
58
57
  import type { SkillOperationContext } from "./handlers/skills.js";
59
58
  import { HostBashProxy } from "./host-bash-proxy.js";
59
+ import { HostCuProxy } from "./host-cu-proxy.js";
60
60
  import { HostFileProxy } from "./host-file-proxy.js";
61
61
  import type { ServerMessage } from "./message-protocol.js";
62
62
  import {
@@ -214,14 +214,18 @@ function makePendingInteractionRegistrar(
214
214
  conversationId,
215
215
  kind: "host_file",
216
216
  });
217
+ } else if (msg.type === "host_cu_request") {
218
+ pendingInteractions.register(msg.requestId, {
219
+ session,
220
+ conversationId,
221
+ kind: "host_cu",
222
+ });
217
223
  }
218
224
  };
219
225
  }
220
226
 
221
227
  export class DaemonServer {
222
228
  private sessions = new Map<string, Session>();
223
- private cuSessions = new Map<string, ComputerUseSession>();
224
- private cuObservationParseSequence = new Map<string, number>();
225
229
  private sessionOptions = new Map<string, SessionCreateOptions>();
226
230
  private sessionCreating = new Map<string, Promise<Session>>();
227
231
  private sharedRequestTimestamps: number[] = [];
@@ -412,11 +416,6 @@ export class DaemonServer {
412
416
  }
413
417
  this.sessions.clear();
414
418
 
415
- for (const cuSession of this.cuSessions.values()) {
416
- cuSession.abort();
417
- }
418
- this.cuSessions.clear();
419
-
420
419
  log.info("Daemon server stopped");
421
420
  }
422
421
 
@@ -563,8 +562,6 @@ export class DaemonServer {
563
562
  private handlerContext(): HandlerContext {
564
563
  return {
565
564
  sessions: this.sessions,
566
- cuSessions: this.cuSessions,
567
- cuObservationParseSequence: this.cuObservationParseSequence,
568
565
  sharedRequestTimestamps: this.sharedRequestTimestamps,
569
566
  debounceTimers: this.configWatcher.timers,
570
567
  suppressConfigReload: this.configWatcher.suppressConfigReload,
@@ -665,9 +662,13 @@ export class DaemonServer {
665
662
  }),
666
663
  );
667
664
  }
665
+ if (!session.isProcessing() || !session.hostCuProxy) {
666
+ session.setHostCuProxy(new HostCuProxy(session.getCurrentSender()));
667
+ }
668
668
  } else if (!session.isProcessing()) {
669
669
  session.setHostBashProxy(undefined);
670
670
  session.setHostFileProxy(undefined);
671
+ session.setHostCuProxy(undefined);
671
672
  }
672
673
  session.setCommandIntent(options?.commandIntent ?? null);
673
674
  session.setTurnChannelContext({
@@ -907,23 +908,15 @@ export class DaemonServer {
907
908
 
908
909
  /**
909
910
  * Look up an active session by ID without creating one.
910
- * Checks both normal sessions and computer-use sessions so the HTTP
911
- * surface-action path is consistent with dispatch.
912
911
  */
913
- findSession(sessionId: string): Session | ComputerUseSession | undefined {
914
- return this.cuSessions.get(sessionId) ?? this.sessions.get(sessionId);
912
+ findSession(sessionId: string): Session | undefined {
913
+ return this.sessions.get(sessionId);
915
914
  }
916
915
 
917
916
  /**
918
917
  * Look up an active session that owns a given surfaceId.
919
- * Falls back across both normal and computer-use sessions.
920
918
  */
921
- findSessionBySurfaceId(
922
- surfaceId: string,
923
- ): Session | ComputerUseSession | undefined {
924
- for (const s of this.cuSessions.values()) {
925
- if (s.surfaceState.has(surfaceId)) return s;
926
- }
919
+ findSessionBySurfaceId(surfaceId: string): Session | undefined {
927
920
  for (const s of this.sessions.values()) {
928
921
  if (s.surfaceState.has(surfaceId)) return s;
929
922
  }
@@ -384,6 +384,7 @@ export function handleToolOutputChunk(
384
384
  type: "tool_output_chunk",
385
385
  chunk: event.chunk,
386
386
  sessionId: deps.ctx.conversationId,
387
+ toolUseId: event.toolUseId,
387
388
  subType: structured.subType,
388
389
  subToolName: structured.subToolName,
389
390
  subToolInput: structured.subToolInput,
@@ -395,6 +396,7 @@ export function handleToolOutputChunk(
395
396
  type: "tool_output_chunk",
396
397
  chunk: event.chunk,
397
398
  sessionId: deps.ctx.conversationId,
399
+ toolUseId: event.toolUseId,
398
400
  });
399
401
  }
400
402
  }
@@ -175,8 +175,7 @@ export async function resolveAssistantAttachments(
175
175
  accumulatedToolContentBlocks,
176
176
  toolContentBlockToolNames,
177
177
  );
178
- // Most recent tool outputs (e.g., final browser screenshot) should win
179
- // the MAX_ASSISTANT_ATTACHMENTS cap over older intermediate screenshots.
178
+ // Most recent tool outputs first so deduplication keeps the latest version.
180
179
  toolDrafts.reverse();
181
180
  const merged = deduplicateDrafts([
182
181
  ...directiveDrafts.drafts,
@@ -465,7 +465,7 @@ function resolvePairCommand(content: string): SlashResolution | null {
465
465
  kind: "unknown",
466
466
  message:
467
467
  "Cannot generate pairing info — no gateway URL is configured and no LAN address was detected.\n\n" +
468
- "Set a public gateway URL with `config set ingress.publicBaseUrl <url>` or the `INGRESS_PUBLIC_BASE_URL` environment variable.",
468
+ "Set a public gateway URL with `config set ingress.publicBaseUrl <url>`.",
469
469
  };
470
470
  }
471
471
 
@@ -202,7 +202,8 @@ export interface SurfaceSessionContext {
202
202
  }>;
203
203
  display?: string;
204
204
  }>;
205
- onEscalateToComputerUse?: (task: string, sourceSessionId: string) => boolean;
205
+ /** Optional proxy for delegating computer-use actions to a connected desktop client. */
206
+ hostCuProxy?: import("./host-cu-proxy.js").HostCuProxy;
206
207
  isProcessing(): boolean;
207
208
  enqueueMessage(
208
209
  content: string,
@@ -931,13 +932,50 @@ export function buildUserFacingLabel(
931
932
 
932
933
  /**
933
934
  * Resolve a proxy tool call that targets a UI surface.
934
- * Handles ui_show, ui_update, ui_dismiss, computer_use_request_control, and app_open.
935
+ * Handles ui_show, ui_update, ui_dismiss, computer_use_* proxy tools, and app_open.
935
936
  */
936
937
  export async function surfaceProxyResolver(
937
938
  ctx: SurfaceSessionContext,
938
939
  toolName: string,
939
940
  input: Record<string, unknown>,
940
941
  ): Promise<ToolExecutionResult> {
942
+ // Route CU proxy tools (all computer_use_* action tools)
943
+ if (toolName.startsWith("computer_use_")) {
944
+ if (!ctx.hostCuProxy) {
945
+ return {
946
+ content: "Computer use is not available — no desktop client connected.",
947
+ isError: true,
948
+ };
949
+ }
950
+
951
+ // Terminal tools resolve immediately without a client round-trip
952
+ if (
953
+ toolName === "computer_use_done" ||
954
+ toolName === "computer_use_respond"
955
+ ) {
956
+ const summary =
957
+ typeof input.summary === "string"
958
+ ? input.summary
959
+ : typeof input.answer === "string"
960
+ ? input.answer
961
+ : "Task complete";
962
+ ctx.hostCuProxy.reset();
963
+ return { content: summary, isError: false };
964
+ }
965
+
966
+ // Record the action and proxy to the connected desktop client
967
+ const reasoning =
968
+ typeof input.reasoning === "string" ? input.reasoning : undefined;
969
+ ctx.hostCuProxy.recordAction(toolName, input, reasoning);
970
+ return ctx.hostCuProxy.request(
971
+ toolName,
972
+ input,
973
+ ctx.conversationId,
974
+ ctx.hostCuProxy.stepCount,
975
+ reasoning,
976
+ );
977
+ }
978
+
941
979
  if (toolName === "ui_show" || toolName === "ui_update") {
942
980
  const caps = ctx.channelCapabilities;
943
981
  if (caps && !caps.supportsDynamicUi) {
@@ -1152,32 +1190,6 @@ export async function surfaceProxyResolver(
1152
1190
  };
1153
1191
  }
1154
1192
 
1155
- if (toolName === "computer_use_request_control") {
1156
- const task =
1157
- typeof input.task === "string"
1158
- ? input.task
1159
- : "Perform the requested task";
1160
- if (!ctx.onEscalateToComputerUse) {
1161
- return {
1162
- content:
1163
- "Computer control escalation is not available in this session.",
1164
- isError: true,
1165
- };
1166
- }
1167
- const success = ctx.onEscalateToComputerUse(task, ctx.conversationId);
1168
- if (!success) {
1169
- return {
1170
- content: "Computer control escalation failed — no active connection.",
1171
- isError: true,
1172
- };
1173
- }
1174
- return {
1175
- content:
1176
- "Computer control activated. The task has been handed off to foreground computer use.",
1177
- isError: false,
1178
- };
1179
- }
1180
-
1181
1193
  if (toolName === "app_open") {
1182
1194
  const appId = input.app_id as string;
1183
1195
  const preview = input.preview as DynamicPageSurfaceData["preview"];
@@ -25,7 +25,6 @@ import type { TrustClass } from "../runtime/actor-trust-resolver.js";
25
25
  import { getEffectiveMode } from "../runtime/session-approval-overrides.js";
26
26
  import { coreAppProxyTools } from "../tools/apps/definitions.js";
27
27
  import { registerSessionSender } from "../tools/browser/browser-screencast.js";
28
- import { requestComputerControlTool } from "../tools/computer-use/request-computer-control.js";
29
28
  import type { ToolExecutor } from "../tools/executor.js";
30
29
  import {
31
30
  getAllToolDefinitions,
@@ -115,16 +114,13 @@ export interface ToolSetupContext extends SurfaceSessionContext {
115
114
 
116
115
  /**
117
116
  * Collect all tool definitions for the agent loop: built-in tools,
118
- * UI surface proxy tools, app proxy tools, and the computer-use
119
- * escalation tool.
117
+ * UI surface proxy tools, and app proxy tools.
120
118
  */
121
119
  export function buildToolDefinitions(): ToolDefinition[] {
122
120
  return [
123
121
  ...getAllToolDefinitions(),
124
122
  ...allUiSurfaceTools.map((t) => t.getDefinition()),
125
123
  ...coreAppProxyTools.map((t) => t.getDefinition()),
126
- // Escalation tool: allows text_qa sessions to hand off to computer use
127
- requestComputerControlTool.getDefinition(),
128
124
  ];
129
125
  }
130
126
 
@@ -566,10 +562,7 @@ const HOST_TOOL_NAMES = new Set([
566
562
  "host_bash",
567
563
  ]);
568
564
  const ASSET_TOOL_NAMES = new Set(["asset_search", "asset_materialize"]);
569
- const CLIENT_CAPABILITY_TOOL_NAMES = new Set([
570
- "app_open",
571
- "computer_use_request_control",
572
- ]);
565
+ const CLIENT_CAPABILITY_TOOL_NAMES = new Set(["app_open"]);
573
566
  const PLATFORM_TOOL_NAMES = new Set(["request_system_permission"]);
574
567
 
575
568
  /**