@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/docs/desktop-browser-cli.md +33 -0
  2. package/openapi.yaml +17 -5
  3. package/package.json +1 -1
  4. package/scripts/smoke-desktop-browser-cli.ts +279 -0
  5. package/src/__tests__/config-schema.test.ts +1 -1
  6. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  7. package/src/__tests__/conversation-error.test.ts +4 -1
  8. package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
  9. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  10. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  11. package/src/__tests__/host-cu-proxy.test.ts +116 -18
  12. package/src/__tests__/list-all-apps.test.ts +40 -8
  13. package/src/__tests__/mcp-auth-routes.test.ts +99 -0
  14. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  15. package/src/__tests__/mcp-health-check.test.ts +32 -32
  16. package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
  17. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  18. package/src/__tests__/skills.test.ts +10 -0
  19. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  20. package/src/acp/session-snapshot.ts +260 -0
  21. package/src/apps/app-store.ts +6 -5
  22. package/src/browser/operations.ts +2 -1
  23. package/src/browser/types.ts +7 -0
  24. package/src/browser/virtual-desktop-target.ts +34 -0
  25. package/src/calls/__tests__/progress-narration.test.ts +23 -0
  26. package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
  27. package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
  28. package/src/calls/progress-narration.ts +15 -2
  29. package/src/calls/voice-control-protocol.ts +34 -4
  30. package/src/calls/voice-session-bridge.ts +23 -0
  31. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  32. package/src/cli/commands/browser.help.ts +43 -0
  33. package/src/cli/commands/browser.ts +52 -8
  34. package/src/cli/commands/plugins.ts +16 -5
  35. package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
  36. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  37. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  38. package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
  39. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  40. package/src/cli/lib/bundled-marketplace.json +13 -0
  41. package/src/cli/lib/install-from-github.ts +17 -18
  42. package/src/cli/lib/install-from-platform.ts +14 -9
  43. package/src/cli/lib/list-installed-plugins.ts +21 -35
  44. package/src/cli/lib/plugin-details.ts +21 -8
  45. package/src/cli/lib/uninstall-plugin.ts +59 -1
  46. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  47. package/src/config/bundled-skills/acp/SKILL.md +1 -1
  48. package/src/config/bundled-skills/acp/TOOLS.json +2 -2
  49. package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
  50. package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
  51. package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
  52. package/src/config/feature-flag-registry.json +2 -2
  53. package/src/config/schemas/__tests__/voice.test.ts +2 -2
  54. package/src/config/schemas/mcp.ts +9 -7
  55. package/src/config/schemas/monitoring.ts +9 -0
  56. package/src/config/schemas/voice.ts +2 -2
  57. package/src/config/skills.ts +13 -21
  58. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  59. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
  60. package/src/daemon/conversation-client-surface.ts +26 -0
  61. package/src/daemon/conversation-runtime-assembly.ts +1 -1
  62. package/src/daemon/conversation-tool-setup.ts +31 -29
  63. package/src/daemon/host-cu-proxy.ts +51 -26
  64. package/src/daemon/host-proxy-preactivation.ts +2 -0
  65. package/src/daemon/mcp-reload-service.ts +7 -0
  66. package/src/daemon/message-types/sync.ts +1 -0
  67. package/src/daemon/providers-setup.ts +4 -0
  68. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  69. package/src/desktop/desktop-automation-lease.test.ts +208 -0
  70. package/src/desktop/desktop-automation-lease.ts +270 -0
  71. package/src/desktop/desktop-browser-client.test.ts +413 -0
  72. package/src/desktop/desktop-browser-client.ts +430 -0
  73. package/src/desktop/desktop-browser-cursor.ts +40 -0
  74. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  75. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  76. package/src/desktop/desktop-browser-operations.ts +130 -0
  77. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  78. package/src/desktop/desktop-chrome-session.ts +58 -8
  79. package/src/desktop/desktop-dependencies.test.ts +29 -0
  80. package/src/desktop/desktop-dependencies.ts +26 -5
  81. package/src/desktop/desktop-display.ts +9 -0
  82. package/src/desktop/desktop-panel-config.ts +14 -1
  83. package/src/desktop/desktop-session-manager.test.ts +86 -3
  84. package/src/desktop/desktop-session-manager.ts +144 -24
  85. package/src/desktop/desktop-stream-bridge.ts +1 -1
  86. package/src/desktop/virtual-desktop-feature.ts +18 -0
  87. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  88. package/src/i18n/__tests__/i18n.test.ts +9 -5
  89. package/src/i18n/messages.ts +13 -4
  90. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  91. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  92. package/src/ipc/assistant-server.ts +15 -1
  93. package/src/ipc/cli-client.ts +13 -5
  94. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
  95. package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
  96. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  97. package/src/live-voice/__tests__/protocol.test.ts +11 -3
  98. package/src/live-voice/__tests__/session-controls.test.ts +34 -0
  99. package/src/live-voice/live-voice-metrics.ts +9 -0
  100. package/src/live-voice/live-voice-session.ts +3 -0
  101. package/src/live-voice/protocol.ts +10 -1
  102. package/src/live-voice/session-controls.ts +28 -0
  103. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  104. package/src/mcp/__tests__/effective-config.test.ts +11 -0
  105. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  106. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  107. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  108. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  109. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  110. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  111. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  112. package/src/mcp/client.ts +64 -16
  113. package/src/mcp/credential-target.ts +105 -0
  114. package/src/mcp/effective-config.ts +42 -15
  115. package/src/mcp/manager.ts +61 -8
  116. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  117. package/src/mcp/mcp-auth-state.ts +38 -6
  118. package/src/mcp/mcp-oauth-provider.ts +101 -33
  119. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  120. package/src/monitoring/mount-watch.ts +348 -0
  121. package/src/monitoring/worker.ts +9 -0
  122. package/src/notifications/__tests__/decision-engine.test.ts +175 -0
  123. package/src/notifications/decision-engine.ts +43 -8
  124. package/src/oauth/seed-providers.ts +11 -0
  125. package/src/onboarding/checkin-event.ts +2 -1
  126. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  127. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  128. package/src/plugins/external-plugin-loader.ts +49 -86
  129. package/src/plugins/installed-plugin-dirs.ts +6 -4
  130. package/src/plugins/mcp-servers.ts +9 -15
  131. package/src/plugins/mtime-cache.ts +2 -1
  132. package/src/plugins/user-loader.ts +2 -1
  133. package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
  134. package/src/providers/inference/endpoint-probe.ts +27 -12
  135. package/src/providers/openai/responses-provider.ts +9 -2
  136. package/src/providers/opencode/client.test.ts +167 -0
  137. package/src/providers/opencode/client.ts +73 -4
  138. package/src/runtime/AGENTS.md +4 -0
  139. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
  140. package/src/runtime/http-server.ts +6 -6
  141. package/src/runtime/routes/__tests__/acp-routes.test.ts +19 -0
  142. package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
  143. package/src/runtime/routes/acp-routes.ts +17 -161
  144. package/src/runtime/routes/browser-context.ts +39 -0
  145. package/src/runtime/routes/browser-routes.ts +16 -47
  146. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  147. package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
  148. package/src/runtime/routes/desktop-setup-routes.ts +7 -5
  149. package/src/runtime/routes/mcp-auth-routes.ts +106 -94
  150. package/src/runtime/routes/plugins-routes.ts +14 -1
  151. package/src/runtime/sync/resource-sync-events.ts +4 -0
  152. package/src/tools/acp/status.test.ts +276 -21
  153. package/src/tools/acp/status.ts +98 -32
  154. package/src/tools/browser/browser-execution.ts +44 -13
  155. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  156. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  157. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  158. package/src/tools/computer-use/definitions.ts +2 -3
  159. package/src/util/abort-reasons.ts +12 -1
  160. package/src/util/plugin-manifest.ts +203 -0
  161. package/src/desktop/desktop-feature.test.ts +0 -18
  162. package/src/desktop/desktop-feature.ts +0 -19
@@ -2146,6 +2146,68 @@ describe("front-door leg tool suppression", () => {
2146
2146
  });
2147
2147
  });
2148
2148
 
2149
+ describe("desktop skill preactivation", () => {
2150
+ async function preactivatedFor(turn: Record<string, unknown>): Promise<{
2151
+ skillIds: string[];
2152
+ proxyInterfaces: unknown[];
2153
+ skillIdsDuringLoop: string[];
2154
+ }> {
2155
+ const skillIds: string[] = [];
2156
+ const proxyInterfaces: unknown[] = [];
2157
+ let skillIdsDuringLoop: string[] = [];
2158
+ const fake = makeFakeConversation({
2159
+ processing: false,
2160
+ runAgentLoop: async () => {
2161
+ skillIdsDuringLoop = [...skillIds];
2162
+ },
2163
+ });
2164
+ Object.assign(fake.conversation, {
2165
+ addPreactivatedSkillId: (id: string) => {
2166
+ skillIds.push(id);
2167
+ },
2168
+ ensureHostProxiesForTurn: (sourceInterface: unknown) => {
2169
+ proxyInterfaces.push(sourceInterface);
2170
+ },
2171
+ });
2172
+ fakeConversation = fake.conversation;
2173
+
2174
+ await startVoiceTurn({
2175
+ ...makeTurnOptions(undefined, "conv-desktop-skills"),
2176
+ userMessageInterface: "macos",
2177
+ ...turn,
2178
+ });
2179
+ await flushMicrotasks();
2180
+ return { skillIds, proxyInterfaces, skillIdsDuringLoop };
2181
+ }
2182
+
2183
+ test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
2184
+ const result = await preactivatedFor({
2185
+ routingLeg: "escalated",
2186
+ macosDesktopSession: true,
2187
+ });
2188
+
2189
+ expect(result.skillIdsDuringLoop).toContain("computer-use");
2190
+ expect(result.proxyInterfaces).toEqual(["macos"]);
2191
+ });
2192
+
2193
+ test("a session from any other client leaves the desktop skills to be loaded", async () => {
2194
+ const result = await preactivatedFor({ routingLeg: "escalated" });
2195
+
2196
+ expect(result.skillIds).toEqual([]);
2197
+ expect(result.proxyInterfaces).toEqual([]);
2198
+ });
2199
+
2200
+ test("the toolless front-door leg is never preactivated", async () => {
2201
+ const result = await preactivatedFor({
2202
+ routingLeg: "front-door",
2203
+ macosDesktopSession: true,
2204
+ });
2205
+
2206
+ expect(result.skillIds).toEqual([]);
2207
+ expect(result.proxyInterfaces).toEqual([]);
2208
+ });
2209
+ });
2210
+
2149
2211
  describe("cutFrontDoorContentAtVerdict", () => {
2150
2212
  test("null when the content carries no verdict token (committed answer)", () => {
2151
2213
  expect(
@@ -16,7 +16,7 @@ import type {
16
16
  ProviderResponse,
17
17
  ToolDefinition,
18
18
  } from "../providers/types.js";
19
- import { createAbortReason } from "../util/abort-reasons.js";
19
+ import { createAbortReason, isAbortReason } from "../util/abort-reasons.js";
20
20
  import { getLogger } from "../util/logger.js";
21
21
 
22
22
  const log = getLogger("voice-progress-narration");
@@ -164,7 +164,10 @@ async function requestBoundedResponse(args: {
164
164
  const timeoutTimer = setTimeout(
165
165
  () =>
166
166
  timeoutController.abort(
167
- createAbortReason("voice_session_aborted", "voice-progress-narration"),
167
+ createAbortReason(
168
+ "voice_progress_narration_timeout",
169
+ "voice-progress-narration",
170
+ ),
168
171
  ),
169
172
  args.timeoutMs,
170
173
  );
@@ -260,9 +263,19 @@ export function createVoiceProgressNarrator(options: {
260
263
  }
261
264
  return trimmed;
262
265
  } catch (error) {
266
+ // Why the beat went unspoken, as one greppable field. A narrator whose
267
+ // budget is below the model's real roundtrip drops every update at the
268
+ // deadline and reads in the logs as a run of unrelated failures; naming
269
+ // the outcome is what makes that run legible as "the feature is off".
270
+ const outcome = isAbortReason(error)
271
+ ? error.kind === "voice_progress_narration_timeout"
272
+ ? "timed_out"
273
+ : "session_ended"
274
+ : "error";
263
275
  log.info(
264
276
  {
265
277
  error,
278
+ outcome,
266
279
  providerResolveMs,
267
280
  totalMs: Math.round(performance.now() - startedAt),
268
281
  timeoutMs: options.config.generationTimeoutMs,
@@ -60,6 +60,18 @@ export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
60
60
  export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
61
61
  const UPDATES_MARKER_PREFIX = "[UPDATES:";
62
62
 
63
+ /**
64
+ * Look session controls for live voice: turn on a screen share
65
+ * ({@link LOOK_SCREEN_MARKER}) or the camera ({@link LOOK_CAMERA_MARKER}) so
66
+ * the call can see what the user means by "take a look". Carried out by the
67
+ * client, like mute and end.
68
+ */
69
+ export const LOOK_SCREEN_MARKER = "[LOOK:SCREEN]";
70
+ export const LOOK_CAMERA_MARKER = "[LOOK:CAMERA]";
71
+ /** Stop showing the call the screen and the camera, whichever is on. */
72
+ export const LOOK_STOP_MARKER = "[LOOK:STOP]";
73
+ const LOOK_MARKER_PREFIX = "[LOOK:";
74
+
63
75
  // ---------------------------------------------------------------------------
64
76
  // Regexes
65
77
  // ---------------------------------------------------------------------------
@@ -82,6 +94,7 @@ const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
82
94
  const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
83
95
  const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
84
96
  const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
97
+ const LOOK_MARKER_REGEX = /\[LOOK:\s*[^\]]*\]/g;
85
98
  const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
86
99
  const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
87
100
 
@@ -209,6 +222,7 @@ export function stripInternalSpeechMarkers(text: string): string {
209
222
  .replace(MINIMIZE_ROOM_MARKER_REGEX, "")
210
223
  .replace(MUTE_MARKER_REGEX, "")
211
224
  .replace(UPDATES_MARKER_REGEX, "")
225
+ .replace(LOOK_MARKER_REGEX, "")
212
226
  .replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
213
227
  .replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
214
228
  return result;
@@ -237,6 +251,7 @@ const CONTROL_MARKER_STRINGS = [
237
251
  MUTE_MARKER,
238
252
  MUTE_MARKER_PREFIX,
239
253
  UPDATES_MARKER_PREFIX,
254
+ LOOK_MARKER_PREFIX,
240
255
  "[GUARDIAN_TIMEOUT]",
241
256
  "[GUARDIAN_UNAVAILABLE]",
242
257
  ];
@@ -252,6 +267,7 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
252
267
  "[USER_INSTRUCTION:",
253
268
  MUTE_MARKER_PREFIX,
254
269
  UPDATES_MARKER_PREFIX,
270
+ LOOK_MARKER_PREFIX,
255
271
  ];
256
272
 
257
273
  const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
@@ -339,15 +355,19 @@ export function createControlMarkerHoldback(
339
355
  /**
340
356
  * A session control a live-voice reply asked for with a terminal marker:
341
357
  * `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
342
- * timed form, `updates` from the progress-cadence markers.
358
+ * timed form, `updates` from the progress-cadence markers, `look_screen` and
359
+ * `look_camera` and `look_stop` from the look markers.
343
360
  */
344
361
  export type SessionControlRequest =
345
362
  | { readonly action: "end" }
346
363
  | { readonly action: "mute"; readonly durationMs?: number }
347
- | { readonly action: "updates"; readonly cadence: "fewer" | "normal" };
364
+ | { readonly action: "updates"; readonly cadence: "fewer" | "normal" }
365
+ | { readonly action: "look_screen" }
366
+ | { readonly action: "look_camera" }
367
+ | { readonly action: "look_stop" };
348
368
 
349
369
  const TERMINAL_SESSION_CONTROL_REGEX =
350
- /(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
370
+ /(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[LOOK:(SCREEN|CAMERA|STOP)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
351
371
 
352
372
  /**
353
373
  * The session control a reply ends with, or null.
@@ -378,7 +398,17 @@ export function parseTerminalSessionControl(
378
398
  cadence: match[2] === "FEWER" ? "fewer" : "normal",
379
399
  };
380
400
  }
381
- const seconds = match[3] === undefined ? NaN : Number(match[3].trim());
401
+ if (match[3] !== undefined) {
402
+ return {
403
+ action:
404
+ match[3] === "SCREEN"
405
+ ? "look_screen"
406
+ : match[3] === "CAMERA"
407
+ ? "look_camera"
408
+ : "look_stop",
409
+ };
410
+ }
411
+ const seconds = match[4] === undefined ? NaN : Number(match[4].trim());
382
412
  if (
383
413
  Number.isFinite(seconds) &&
384
414
  seconds > 0 &&
@@ -31,6 +31,7 @@ import {
31
31
  import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
32
32
  import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
33
33
  import { getOrCreateConversation } from "../daemon/conversation-store.js";
34
+ import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
34
35
  import type { TrustContext } from "../daemon/trust-context-types.js";
35
36
  import {
36
37
  newestPersistedSightFrame,
@@ -464,6 +465,14 @@ export interface VoiceTurnOptions {
464
465
  * path has always used.
465
466
  */
466
467
  actorFallbackSuppressed?: boolean;
468
+ /**
469
+ * Whether the session was opened from the macOS desktop client. A leg that
470
+ * can run tools then starts with the desktop skills (computer use, app
471
+ * control, screen annotation) already active, as a typed turn from that
472
+ * client does. The turn interface cannot answer this: every live-voice
473
+ * session reports `macos` for its channel capabilities, iOS included.
474
+ */
475
+ macosDesktopSession?: boolean;
467
476
  /** Whether this is an inbound call (no outbound task). */
468
477
  isInbound: boolean;
469
478
  /** The outbound call task, if any. */
@@ -2016,6 +2025,20 @@ export async function startVoiceTurn(
2016
2025
  (needsImagePin
2017
2026
  ? VOICE_IMAGE_PROFILE
2018
2027
  : (conversationProfile?.profile ?? null));
2028
+ if (opts.macosDesktopSession === true && !frontDoorToolsSuppressed) {
2029
+ const sourceInterface = turnInterfaceContext.userMessageInterface;
2030
+ const sourceActorPrincipalId =
2031
+ voiceTurnValues.actorPrincipalId ?? undefined;
2032
+ conversation.ensureHostProxiesForTurn(
2033
+ sourceInterface,
2034
+ sourceActorPrincipalId,
2035
+ );
2036
+ preactivateHostProxySkills(
2037
+ conversation,
2038
+ sourceInterface,
2039
+ sourceActorPrincipalId,
2040
+ );
2041
+ }
2019
2042
  await conversation.runAgentLoop(persistedContent, messageId, {
2020
2043
  onEvent: (msg: AssistantEvent) => {
2021
2044
  if (msg.type === "assistant_turn_start") {
@@ -67,9 +67,8 @@ mock.module("../../../util/logger.js", () => ({
67
67
  // Import module under test (after mocks)
68
68
  // ---------------------------------------------------------------------------
69
69
 
70
- const { formatBrowserStatusLines, registerBrowserCommand } = await import(
71
- "../browser.js"
72
- );
70
+ const { formatBrowserStatusLines, registerBrowserCommand } =
71
+ await import("../browser.js");
73
72
 
74
73
  // ---------------------------------------------------------------------------
75
74
  // Test helper
@@ -641,3 +640,85 @@ describe("error exit codes", () => {
641
640
  expect(exitCode).toBe(0);
642
641
  });
643
642
  });
643
+
644
+ for (const desktopFlag of ["--virtual-desktop", "--desktop"]) {
645
+ describe(`virtual desktop routing via ${desktopFlag}`, () => {
646
+ test("routes semantic actions through IPC with conversation and element identity", async () => {
647
+ process.env.__CONVERSATION_ID = "conv-desktop";
648
+ const result = await runCommand([
649
+ "browser",
650
+ desktopFlag,
651
+ "--json",
652
+ "click",
653
+ "--element-id",
654
+ "e7",
655
+ ]);
656
+ expect(result.exitCode).toBe(0);
657
+ expect(lastIpcCall?.method).toBe("browser_execute");
658
+ expect(lastBody()).toMatchObject({
659
+ desktop: true,
660
+ conversationId: "conv-desktop",
661
+ operation: "click",
662
+ input: { element_id: "e7" },
663
+ });
664
+ expect(
665
+ (lastBody().input as Record<string, unknown>).desktop,
666
+ ).toBeUndefined();
667
+ });
668
+
669
+ test.each([
670
+ { args: ["list"] },
671
+ { args: ["select", "--tab-id", "7"] },
672
+ { args: ["new"] },
673
+ { args: ["close", "--tab-id", "7"] },
674
+ ])("routes desktop tabs $args through IPC", async ({ args }) => {
675
+ mockIpcResult = { ok: true, result: { ok: true, tabs: [] } };
676
+ const result = await runCommand([
677
+ "browser",
678
+ desktopFlag,
679
+ "--json",
680
+ "tabs",
681
+ ...args,
682
+ ]);
683
+ expect(result.exitCode).toBe(0);
684
+ expect(lastIpcCall?.method).toBe("browser_tabs");
685
+ expect(lastBody()).toMatchObject({ desktop: true, command: args[0] });
686
+ });
687
+
688
+ test.each([
689
+ { flags: ["--browser-mode", "extension"] },
690
+ { flags: ["--target-client-id", "client-123"] },
691
+ ])("rejects conflicting desktop targets before IPC", async ({ flags }) => {
692
+ const result = await runCommand([
693
+ "browser",
694
+ desktopFlag,
695
+ ...flags,
696
+ "tabs",
697
+ "list",
698
+ ]);
699
+ expect(result.exitCode).not.toBe(0);
700
+ expect(lastIpcCall).toBeNull();
701
+ });
702
+ });
703
+ }
704
+
705
+ test.each([
706
+ { args: ["list"] },
707
+ { args: ["select", "--tab-id", "1"] },
708
+ { args: ["new", "--url", "https://example.com"] },
709
+ { args: ["close", "--tab-id", "1"] },
710
+ ])(
711
+ "tab command $args forwards an explicit backend override",
712
+ async ({ args }) => {
713
+ mockIpcResult = { ok: true, result: { ok: true, tabs: [] } };
714
+ await runCommand([
715
+ "browser",
716
+ "--browser-mode",
717
+ "extension",
718
+ "tabs",
719
+ ...args,
720
+ ]);
721
+ expect(lastIpcCall?.method).toBe("browser_tabs");
722
+ expect(lastBody().browserMode).toBe("extension");
723
+ },
724
+ );
@@ -96,6 +96,15 @@ export const browserHelp: CliCommandHelp = {
96
96
  name: "browser",
97
97
  description: "Control the browser via the running assistant.",
98
98
  options: [
99
+ {
100
+ flags: "--desktop",
101
+ description: "Compatibility alias for --virtual-desktop.",
102
+ },
103
+ {
104
+ flags: "--virtual-desktop",
105
+ description:
106
+ "Use Chrome in the platform-hosted virtual desktop with visible cursor feedback.",
107
+ },
99
108
  {
100
109
  flags: "--session <id>",
101
110
  description: "Session ID to preserve browser state across invocations.",
@@ -118,6 +127,38 @@ export const browserHelp: CliCommandHelp = {
118
127
  ],
119
128
  helpText: `
120
129
  Browser operations are executed through the running assistant.
130
+ Virtual desktop is available only for platform-hosted assistants.
131
+ In the web client, automatic selection prefers virtual desktop Chrome when
132
+ its feature is enabled and the conversation has an
133
+ identified guardian. The macOS and Windows apps keep their existing browser
134
+ selection and fallback behavior; use --virtual-desktop to select it there.
135
+ Existing sessions and explicit backend/client choices take priority.
136
+ --browser-mode local (alias playwright) means the assistant's Playwright browser,
137
+ not the user's Chrome. --desktop remains an alias for --virtual-desktop.
138
+ Use --virtual-desktop for Chrome in the Desktop panel.
139
+ Run navigate --url <url> directly to open a page.
140
+ The assistant-desktop feature flag and completed installation are required.
141
+ The first browser command installs missing components, starts Chrome, and completes
142
+ the requested action in one call. Allow up to 600 seconds for this command when
143
+ using bash (timeout_seconds: 600). Opening the Desktop panel also starts
144
+ the same installation and shows progress. No separate setup command is needed.
145
+ Report other availability errors without switching to a personal computer.
146
+ The assistant manages Chrome startup. Do not launch a separate desktop stack,
147
+ install desktop packages, or drive webpages with shell-level xdotool.
148
+ With --virtual-desktop, screenshot captures the page directly from Chrome over CDP
149
+ as a color JPEG. Use --output to save it and file_read to view it. Do not use
150
+ shell-level xwd or custom screenshot conversion scripts.
151
+ Use snapshot for current element IDs and tabs list for current tab IDs.
152
+ Refresh the snapshot after navigation, tab changes or stale-element errors.
153
+ Inspect the result before retrying a failed action; it may have happened.
154
+ A purple page pointer shows CDP mouse movement in the stream. Browser toolbar
155
+ controls, native dialogs and other apps are outside this CLI's page controls.
156
+ Do not combine --virtual-desktop with personal browser targets, other browser modes
157
+ or --use-active-tab. Download waiting is unavailable on this target.
158
+ Users can interact directly with the expanded desktop. If they ask you to pause,
159
+ stop and wait. Take a fresh snapshot before resuming. Closing the viewer does not
160
+ end automation. Run assistant browser --virtual-desktop detach when finished or
161
+ blocked, including before asking a question. Chrome stays running.
121
162
  Each subcommand maps to a browser operation and communicates
122
163
  with the assistant process.
123
164
 
@@ -131,6 +172,8 @@ local. Useful for debugging or when deterministic backend selection
131
172
  is required.
132
173
 
133
174
  Examples:
175
+ $ assistant browser --virtual-desktop status
176
+ $ assistant browser --virtual-desktop screenshot --output /tmp/desktop-page.jpg
134
177
  $ assistant browser navigate --url https://example.com
135
178
  $ assistant browser snapshot
136
179
  $ assistant browser click --selector "#login"
@@ -28,6 +28,8 @@ import { registerCommand } from "../lib/register-command.js";
28
28
  import { log } from "../logger.js";
29
29
  import { browserHelp, toKebab } from "./browser.help.js";
30
30
 
31
+ const BROWSER_REQUEST_TIMEOUT_MS = 10 * 60_000;
32
+
31
33
  // ── Naming helpers ───────────────────────────────────────────────────
32
34
 
33
35
  /**
@@ -116,6 +118,7 @@ function attachOperationAction(
116
118
 
117
119
  subcmd.action(async (opts: Record<string, unknown>) => {
118
120
  const parentOpts = browser.opts() as {
121
+ desktop?: boolean;
119
122
  session?: string;
120
123
  json?: boolean;
121
124
  browserMode?: string;
@@ -130,6 +133,8 @@ function attachOperationAction(
130
133
  // targetClientId) and screenshot ergonomics (output).
131
134
  const input: Record<string, unknown> = {};
132
135
  const excludeKeys = new Set([
136
+ "desktop",
137
+ "virtualDesktop",
133
138
  "session",
134
139
  "json",
135
140
  "output",
@@ -163,9 +168,7 @@ function attachOperationAction(
163
168
  }
164
169
  }
165
170
 
166
- // Browser operations can be long-running (page loads, auth
167
- // challenges, downloads up to 120s, etc.), so use a generous
168
- // IPC timeout that exceeds any server-side operation timeout.
171
+ // First use can include virtual desktop installation and Chrome startup.
169
172
  const ipcResult = await cliIpcCall<BrowserExecuteResult>(
170
173
  "browser_execute",
171
174
  {
@@ -173,10 +176,11 @@ function attachOperationAction(
173
176
  operation: meta.operation,
174
177
  input,
175
178
  sessionId,
179
+ ...(parentOpts.desktop ? { desktop: true } : {}),
176
180
  ...(conversationId ? { conversationId } : {}),
177
181
  },
178
182
  },
179
- { timeoutMs: 180_000 },
183
+ { timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
180
184
  );
181
185
 
182
186
  if (!ipcResult.ok) {
@@ -328,6 +332,22 @@ export function registerBrowserCommand(program: Command): void {
328
332
  description: browserHelp.description,
329
333
  build: (browser) => {
330
334
  applyCommandHelp(browser, browserHelp);
335
+ browser.hook("preAction", () => {
336
+ if (browser.opts().virtualDesktop) {
337
+ browser.setOptionValue("desktop", true);
338
+ }
339
+ const options = browser.opts();
340
+ if (
341
+ options.desktop &&
342
+ (options.targetClientId ||
343
+ (options.browserMode &&
344
+ !["auto", "cdp-inspect"].includes(options.browserMode)))
345
+ ) {
346
+ throw new Error(
347
+ "--virtual-desktop cannot be combined with a personal browser target or another browser mode",
348
+ );
349
+ }
350
+ });
331
351
 
332
352
  // Attach one action per browser operation
333
353
  for (const meta of BROWSER_OPERATION_META) {
@@ -339,6 +359,8 @@ export function registerBrowserCommand(program: Command): void {
339
359
 
340
360
  subcommand(tabs, "list").action(async (opts: { pretty?: boolean }) => {
341
361
  const parentOpts = browser.opts() as {
362
+ desktop?: boolean;
363
+ browserMode?: string;
342
364
  session?: string;
343
365
  json?: boolean;
344
366
  targetClientId?: string;
@@ -364,11 +386,15 @@ export function registerBrowserCommand(program: Command): void {
364
386
  body: {
365
387
  command: "list",
366
388
  sessionId,
389
+ ...(parentOpts.desktop ? { desktop: true } : {}),
390
+ ...(parentOpts.browserMode
391
+ ? { browserMode: parentOpts.browserMode }
392
+ : {}),
367
393
  ...(conversationId ? { conversationId } : {}),
368
394
  ...(targetClientId ? { targetClientId } : {}),
369
395
  },
370
396
  },
371
- { timeoutMs: 30_000 },
397
+ { timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
372
398
  );
373
399
 
374
400
  if (!ipcResult.ok) {
@@ -409,6 +435,8 @@ export function registerBrowserCommand(program: Command): void {
409
435
 
410
436
  subcommand(tabs, "select").action(async (opts: { tabId: string }) => {
411
437
  const parentOpts = browser.opts() as {
438
+ desktop?: boolean;
439
+ browserMode?: string;
412
440
  session?: string;
413
441
  json?: boolean;
414
442
  targetClientId?: string;
@@ -425,12 +453,16 @@ export function registerBrowserCommand(program: Command): void {
425
453
  body: {
426
454
  command: "select",
427
455
  sessionId,
456
+ ...(parentOpts.desktop ? { desktop: true } : {}),
457
+ ...(parentOpts.browserMode
458
+ ? { browserMode: parentOpts.browserMode }
459
+ : {}),
428
460
  tabId,
429
461
  ...(conversationId ? { conversationId } : {}),
430
462
  ...(targetClientId ? { targetClientId } : {}),
431
463
  },
432
464
  },
433
- { timeoutMs: 30_000 },
465
+ { timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
434
466
  );
435
467
 
436
468
  if (!ipcResult.ok) {
@@ -456,6 +488,8 @@ export function registerBrowserCommand(program: Command): void {
456
488
 
457
489
  subcommand(tabs, "new").action(async (opts: { url?: string }) => {
458
490
  const parentOpts = browser.opts() as {
491
+ desktop?: boolean;
492
+ browserMode?: string;
459
493
  session?: string;
460
494
  json?: boolean;
461
495
  targetClientId?: string;
@@ -475,12 +509,16 @@ export function registerBrowserCommand(program: Command): void {
475
509
  body: {
476
510
  command: "new",
477
511
  sessionId,
512
+ ...(parentOpts.desktop ? { desktop: true } : {}),
513
+ ...(parentOpts.browserMode
514
+ ? { browserMode: parentOpts.browserMode }
515
+ : {}),
478
516
  ...(opts.url ? { url: opts.url } : {}),
479
517
  ...(conversationId ? { conversationId } : {}),
480
518
  ...(targetClientId ? { targetClientId } : {}),
481
519
  },
482
520
  },
483
- { timeoutMs: 30_000 },
521
+ { timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
484
522
  );
485
523
 
486
524
  if (!ipcResult.ok) {
@@ -512,6 +550,8 @@ export function registerBrowserCommand(program: Command): void {
512
550
 
513
551
  subcommand(tabs, "close").action(async (opts: { tabId: string }) => {
514
552
  const parentOpts = browser.opts() as {
553
+ desktop?: boolean;
554
+ browserMode?: string;
515
555
  session?: string;
516
556
  json?: boolean;
517
557
  targetClientId?: string;
@@ -532,12 +572,16 @@ export function registerBrowserCommand(program: Command): void {
532
572
  body: {
533
573
  command: "close",
534
574
  sessionId,
575
+ ...(parentOpts.desktop ? { desktop: true } : {}),
576
+ ...(parentOpts.browserMode
577
+ ? { browserMode: parentOpts.browserMode }
578
+ : {}),
535
579
  tabId,
536
580
  ...(conversationId ? { conversationId } : {}),
537
581
  ...(targetClientId ? { targetClientId } : {}),
538
582
  },
539
583
  },
540
- { timeoutMs: 30_000 },
584
+ { timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
541
585
  );
542
586
 
543
587
  if (!ipcResult.ok) {
@@ -53,6 +53,7 @@ import {
53
53
  PluginAlreadyInStateException,
54
54
  PluginDirectoryNotFoundError,
55
55
  } from "../lib/toggle-plugin.js";
56
+ import type { PluginUninstallWarningKey } from "../lib/uninstall-plugin.js";
56
57
  import type { PluginUpgradeResult } from "../lib/upgrade-plugin.js";
57
58
  import { getCliLogger } from "../logger.js";
58
59
  import { PLUGINS_SEARCH_INSTALL_HINT, pluginsHelp } from "./plugins.help.js";
@@ -666,11 +667,16 @@ export function registerPluginsCommand(program: Command): void {
666
667
  // when the daemon is unreachable (a transport error carries no
667
668
  // `statusCode`); an operator can still uninstall while it's stopped,
668
669
  // and `shutdown` then runs in this process, the only one available.
669
- const daemon = await cliIpcCall<{ name: string; target: string }>(
670
- "plugins_uninstall",
671
- { pathParams: { name } },
672
- );
673
- let result: { name: string; target: string };
670
+ const daemon = await cliIpcCall<{
671
+ name: string;
672
+ target: string;
673
+ warnings?: PluginUninstallWarningKey[];
674
+ }>("plugins_uninstall", { pathParams: { name } });
675
+ let result: {
676
+ name: string;
677
+ target: string;
678
+ warnings?: PluginUninstallWarningKey[];
679
+ };
674
680
  if (daemon.ok && daemon.result) {
675
681
  result = daemon.result;
676
682
  } else if (daemon.statusCode === undefined) {
@@ -693,6 +699,11 @@ export function registerPluginsCommand(program: Command): void {
693
699
  console.log(
694
700
  `Uninstalled plugin "${result.name}" from ${result.target}`,
695
701
  );
702
+ for (const warning of result.warnings ?? []) {
703
+ console.warn(
704
+ `Warning: ${libs.uninstall.resolvePluginUninstallWarning(warning)}`,
705
+ );
706
+ }
696
707
  } catch (err) {
697
708
  if (err instanceof libs.installGitHub.InvalidPluginNameError) {
698
709
  console.error(err.message);