@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/docs/desktop-browser-cli.md +33 -0
  2. package/openapi.yaml +17 -5
  3. package/package.json +1 -1
  4. package/scripts/smoke-desktop-browser-cli.ts +279 -0
  5. package/src/__tests__/config-schema.test.ts +1 -1
  6. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  7. package/src/__tests__/conversation-error.test.ts +4 -1
  8. package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
  9. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  10. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  11. package/src/__tests__/host-cu-proxy.test.ts +116 -18
  12. package/src/__tests__/list-all-apps.test.ts +40 -8
  13. package/src/__tests__/mcp-auth-routes.test.ts +99 -0
  14. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  15. package/src/__tests__/mcp-health-check.test.ts +32 -32
  16. package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
  17. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  18. package/src/__tests__/skills.test.ts +10 -0
  19. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  20. package/src/apps/app-store.ts +6 -5
  21. package/src/browser/operations.ts +2 -1
  22. package/src/browser/types.ts +7 -0
  23. package/src/browser/virtual-desktop-target.ts +34 -0
  24. package/src/calls/__tests__/progress-narration.test.ts +23 -0
  25. package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
  26. package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
  27. package/src/calls/progress-narration.ts +15 -2
  28. package/src/calls/voice-control-protocol.ts +34 -4
  29. package/src/calls/voice-session-bridge.ts +23 -0
  30. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  31. package/src/cli/commands/browser.help.ts +43 -0
  32. package/src/cli/commands/browser.ts +52 -8
  33. package/src/cli/commands/plugins.ts +16 -5
  34. package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
  35. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  36. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  37. package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
  38. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  39. package/src/cli/lib/bundled-marketplace.json +13 -0
  40. package/src/cli/lib/install-from-github.ts +17 -18
  41. package/src/cli/lib/install-from-platform.ts +14 -9
  42. package/src/cli/lib/list-installed-plugins.ts +21 -35
  43. package/src/cli/lib/plugin-details.ts +21 -8
  44. package/src/cli/lib/uninstall-plugin.ts +59 -1
  45. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  46. package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
  47. package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
  48. package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
  49. package/src/config/feature-flag-registry.json +2 -2
  50. package/src/config/schemas/__tests__/voice.test.ts +2 -2
  51. package/src/config/schemas/mcp.ts +9 -7
  52. package/src/config/schemas/monitoring.ts +9 -0
  53. package/src/config/schemas/voice.ts +2 -2
  54. package/src/config/skills.ts +13 -21
  55. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  56. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
  57. package/src/daemon/conversation-client-surface.ts +26 -0
  58. package/src/daemon/conversation-runtime-assembly.ts +1 -1
  59. package/src/daemon/conversation-tool-setup.ts +31 -29
  60. package/src/daemon/host-cu-proxy.ts +51 -26
  61. package/src/daemon/host-proxy-preactivation.ts +2 -0
  62. package/src/daemon/mcp-reload-service.ts +7 -0
  63. package/src/daemon/message-types/sync.ts +1 -0
  64. package/src/daemon/providers-setup.ts +4 -0
  65. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  66. package/src/desktop/desktop-automation-lease.test.ts +208 -0
  67. package/src/desktop/desktop-automation-lease.ts +270 -0
  68. package/src/desktop/desktop-browser-client.test.ts +413 -0
  69. package/src/desktop/desktop-browser-client.ts +430 -0
  70. package/src/desktop/desktop-browser-cursor.ts +40 -0
  71. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  72. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  73. package/src/desktop/desktop-browser-operations.ts +130 -0
  74. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  75. package/src/desktop/desktop-chrome-session.ts +58 -8
  76. package/src/desktop/desktop-dependencies.test.ts +29 -0
  77. package/src/desktop/desktop-dependencies.ts +26 -5
  78. package/src/desktop/desktop-display.ts +9 -0
  79. package/src/desktop/desktop-panel-config.ts +14 -1
  80. package/src/desktop/desktop-session-manager.test.ts +86 -3
  81. package/src/desktop/desktop-session-manager.ts +144 -24
  82. package/src/desktop/desktop-stream-bridge.ts +1 -1
  83. package/src/desktop/virtual-desktop-feature.ts +18 -0
  84. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  85. package/src/i18n/__tests__/i18n.test.ts +9 -5
  86. package/src/i18n/messages.ts +13 -4
  87. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  88. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  89. package/src/ipc/assistant-server.ts +15 -1
  90. package/src/ipc/cli-client.ts +13 -5
  91. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
  92. package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
  93. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  94. package/src/live-voice/__tests__/protocol.test.ts +11 -3
  95. package/src/live-voice/__tests__/session-controls.test.ts +34 -0
  96. package/src/live-voice/live-voice-metrics.ts +9 -0
  97. package/src/live-voice/live-voice-session.ts +3 -0
  98. package/src/live-voice/protocol.ts +10 -1
  99. package/src/live-voice/session-controls.ts +28 -0
  100. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  101. package/src/mcp/__tests__/effective-config.test.ts +11 -0
  102. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  103. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  104. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  105. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  106. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  107. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  108. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  109. package/src/mcp/client.ts +64 -16
  110. package/src/mcp/credential-target.ts +105 -0
  111. package/src/mcp/effective-config.ts +42 -15
  112. package/src/mcp/manager.ts +61 -8
  113. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  114. package/src/mcp/mcp-auth-state.ts +38 -6
  115. package/src/mcp/mcp-oauth-provider.ts +101 -33
  116. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  117. package/src/monitoring/mount-watch.ts +348 -0
  118. package/src/monitoring/worker.ts +9 -0
  119. package/src/oauth/seed-providers.ts +11 -0
  120. package/src/onboarding/checkin-event.ts +2 -1
  121. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  122. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  123. package/src/plugins/external-plugin-loader.ts +49 -86
  124. package/src/plugins/installed-plugin-dirs.ts +6 -4
  125. package/src/plugins/mcp-servers.ts +9 -15
  126. package/src/plugins/mtime-cache.ts +2 -1
  127. package/src/plugins/user-loader.ts +2 -1
  128. package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
  129. package/src/providers/inference/endpoint-probe.ts +27 -12
  130. package/src/providers/openai/responses-provider.ts +9 -2
  131. package/src/providers/opencode/client.test.ts +167 -0
  132. package/src/providers/opencode/client.ts +73 -4
  133. package/src/runtime/AGENTS.md +4 -0
  134. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
  135. package/src/runtime/http-server.ts +6 -6
  136. package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
  137. package/src/runtime/routes/browser-context.ts +39 -0
  138. package/src/runtime/routes/browser-routes.ts +16 -47
  139. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  140. package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
  141. package/src/runtime/routes/desktop-setup-routes.ts +7 -5
  142. package/src/runtime/routes/mcp-auth-routes.ts +106 -94
  143. package/src/runtime/routes/plugins-routes.ts +14 -1
  144. package/src/runtime/sync/resource-sync-events.ts +4 -0
  145. package/src/tools/browser/browser-execution.ts +44 -13
  146. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  147. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  148. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  149. package/src/tools/computer-use/definitions.ts +2 -3
  150. package/src/util/abort-reasons.ts +12 -1
  151. package/src/util/plugin-manifest.ts +203 -0
  152. package/src/desktop/desktop-feature.test.ts +0 -18
  153. package/src/desktop/desktop-feature.ts +0 -19
@@ -540,8 +540,8 @@
540
540
  "id": "assistant-desktop",
541
541
  "scope": "assistant",
542
542
  "key": "assistant-desktop",
543
- "label": "Assistant Desktop",
544
- "description": "Streams an interactive desktop from a containerized assistant pod to the web client over /v1/desktop/stream. Off: no desktop affordance is rendered. On: the chat header offers a desktop panel that views and controls the pod's display.",
543
+ "label": "Virtual desktop",
544
+ "description": "Streams an interactive virtual desktop from a platform-hosted, containerized assistant over /v1/desktop/stream. Requires platform hosting; self-hosted assistants are excluded. Off: the virtual desktop is hidden. On: the chat header offers its viewer and controls.",
545
545
  "defaultEnabled": false
546
546
  }
547
547
  ]
@@ -9,7 +9,7 @@ const PROGRESS_DEFAULTS = {
9
9
  maxSilenceMs: 35_000,
10
10
  longOpMs: 15_000,
11
11
  minGapMs: 6_000,
12
- generationTimeoutMs: 1_500,
12
+ generationTimeoutMs: 5_000,
13
13
  };
14
14
 
15
15
  const FRONT_MODEL_DEFAULTS = {
@@ -75,7 +75,7 @@ describe("VoiceFrontModelConfigSchema", () => {
75
75
  expect(parsed.progress.maxSilenceMs).toBe(35_000);
76
76
  expect(parsed.progress.longOpMs).toBe(15_000);
77
77
  expect(parsed.progress.minGapMs).toBe(6_000);
78
- expect(parsed.progress.generationTimeoutMs).toBe(1_500);
78
+ expect(parsed.progress.generationTimeoutMs).toBe(5_000);
79
79
  });
80
80
 
81
81
  test("a stale maxPerTurn key is stripped, not rejected", () => {
@@ -55,9 +55,7 @@ const McpSseTransportSchema = z
55
55
  .optional()
56
56
  .describe("Custom HTTP headers sent with SSE requests"),
57
57
  })
58
- .describe(
59
- "SSE transport: connects to an MCP server over Server-Sent Events",
60
- );
58
+ .describe("SSE transport: connects to an MCP server over Server-Sent Events");
61
59
 
62
60
  const McpStreamableHttpTransportSchema = z
63
61
  .object({
@@ -114,10 +112,14 @@ export type McpConfig = z.infer<typeof McpConfigSchema>;
114
112
  */
115
113
  export type McpServerSource = "workspace" | "plugin";
116
114
 
117
- /** A server config with its origin resolved. */
118
- export interface ResolvedMcpServerConfig extends McpServerConfig {
119
- readonly source: McpServerSource;
120
- }
115
+ /** A server config with its runtime-only origin and credential identity. */
116
+ export type ResolvedMcpServerConfig =
117
+ | (McpServerConfig & { readonly source: "workspace" })
118
+ | (McpServerConfig & {
119
+ readonly source: "plugin";
120
+ readonly pluginName: string;
121
+ readonly serverKey: string;
122
+ });
121
123
 
122
124
  /** The MCP config the daemon runs: both origins, every server attributed. */
123
125
  export interface ResolvedMcpConfig {
@@ -75,6 +75,15 @@ export const MonitoringConfigSchema = z
75
75
  .describe(
76
76
  "Minimum interval between file-descriptor warnings while usage stays over the threshold, in milliseconds. Crossing the threshold always warns immediately; this only throttles the repeats.",
77
77
  ),
78
+ mountWatchIntervalMs: z
79
+ .number({ error: "monitoring.mountWatchIntervalMs must be a number" })
80
+ .int("monitoring.mountWatchIntervalMs must be an integer")
81
+ .min(1_000, "monitoring.mountWatchIntervalMs must be at least 1000ms")
82
+ .max(60_000, "monitoring.mountWatchIntervalMs must be <= 60000ms")
83
+ .default(10_000)
84
+ .describe(
85
+ "How often the monitor diffs the assistant container's mount table (/proc/self/mountinfo plus statfs of /workspace and /data), in milliseconds. Logs only when the watched mounts change. Default 10 seconds so a virtiofs bind drop is timestamped without a line per poll.",
86
+ ),
78
87
  pluginSourceScanIntervalMs: z
79
88
  .number({
80
89
  error: "monitoring.pluginSourceScanIntervalMs must be a number",
@@ -74,9 +74,9 @@ export const VoiceProgressConfigSchema = z
74
74
  .positive(
75
75
  "voice.frontModel.progress.generationTimeoutMs must be a positive integer",
76
76
  )
77
- .default(1_500)
77
+ .default(5_000)
78
78
  .describe(
79
- "Budget (ms) for LLM-generated progress text. Not latency-critical: it speaks into dead air",
79
+ "Budget (ms) for LLM-generated progress text. Not latency-critical: it speaks into dead air, so the budget is set to cover the narrator's real roundtrip rather than to be tight. A budget that only sometimes covers that roundtrip thins narration out rather than speeding it up, and it thins out exactly where the updates matter most, since the slow beats are the ones with the longest silence behind them (see endpointDecisionTimeoutMs for the same failure mode). Kept under progress.minGapMs so an update that spends the whole budget still lands before the next one is due",
80
80
  ),
81
81
  })
82
82
  // The heartbeat is checked when the idle tick finds the turn silent, so a
@@ -39,6 +39,10 @@ import {
39
39
  getWorkspacePluginsDir,
40
40
  getWorkspaceSkillsDir,
41
41
  } from "../util/platform.js";
42
+ import {
43
+ hasPluginManifest,
44
+ readPluginManifest,
45
+ } from "../util/plugin-manifest.js";
42
46
  import { stripCommentLines } from "../util/strip-comment-lines.js";
43
47
  import { isAssistantFeatureFlagEnabled } from "./assistant-feature-flags.js";
44
48
  import { getConfig } from "./loader.js";
@@ -734,8 +738,7 @@ function discoverSkillDirectories(skillsDir: string): string[] {
734
738
  }
735
739
 
736
740
  /**
737
- * Whether `pluginDir` carries a plugin manifest the runtime can load: a
738
- * parseable `package.json` with a non-empty string `name`. This mirrors the
741
+ * Whether `pluginDir` carries a plugin manifest the runtime can load. This mirrors the
739
742
  * external plugin loader (`buildPluginFromDir`), which builds a plugin from
740
743
  * any such directory and derives the plugin's identity from `package.json`
741
744
  * `name` — it imposes no match between that `name` and the directory name.
@@ -745,31 +748,20 @@ function discoverSkillDirectories(skillsDir: string): string[] {
745
748
  * requiring the two to match would silently drop the resident skills of every
746
749
  * such plugin even though the runtime loads its hooks and tools fine.
747
750
  *
748
- * The caller is responsible for the missing-`package.json` case (it emits a
751
+ * The caller is responsible for the missing-manifest case (it emits a
749
752
  * diagnostic warning); this function only judges a manifest that is present.
750
753
  */
751
754
  function hasLoadablePluginManifest(pluginDir: string): boolean {
752
- const manifestPath = join(pluginDir, "package.json");
753
- if (!existsSync(manifestPath)) {
754
- return false;
755
- }
756
- let parsed: unknown;
757
755
  try {
758
- parsed = JSON.parse(readFileSync(manifestPath, "utf-8"));
756
+ readPluginManifest(pluginDir);
757
+ return true;
759
758
  } catch (err) {
760
759
  log.warn(
761
- { err, manifestPath },
762
- "Skipping plugin dir with unparseable package.json for resident skills",
760
+ { err, pluginDir },
761
+ "Skipping plugin dir with an invalid manifest for resident skills",
763
762
  );
764
763
  return false;
765
764
  }
766
- return (
767
- typeof parsed === "object" &&
768
- parsed !== null &&
769
- "name" in parsed &&
770
- typeof (parsed as { name: unknown }).name === "string" &&
771
- (parsed as { name: string }).name.length > 0
772
- );
773
765
  }
774
766
 
775
767
  /**
@@ -885,15 +877,15 @@ function discoverInstalledPluginResidentSkills(): SkillSummary[] {
885
877
  }
886
878
  const pluginDir = join(pluginsDir, entry.name);
887
879
 
888
- // A directory under `plugins/` with no `package.json` is not a plugin the
880
+ // A directory under `plugins/` with no supported manifest is not a plugin the
889
881
  // runtime can load, so its skills are never surfaced. This is an easy
890
882
  // footgun — a plugin dropped in without its manifest looks installed but
891
883
  // silently contributes nothing — so warn loudly with the path rather than
892
884
  // skipping in silence, to make the misconfiguration diagnosable.
893
- if (!existsSync(join(pluginDir, "package.json"))) {
885
+ if (!hasPluginManifest(pluginDir)) {
894
886
  log.warn(
895
887
  { pluginDir },
896
- "Plugin directory is missing package.json skipping; its skills will not be available. Add a package.json with a `name`.",
888
+ "Plugin directory is missing package.json and plugin.json; its skills will not be available.",
897
889
  );
898
890
  continue;
899
891
  }
@@ -6,6 +6,7 @@
6
6
 
7
7
  import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
8
8
 
9
+ import { setOverridesForTesting } from "../../__tests__/feature-flag-test-helpers.js";
9
10
  import * as configLoader from "../../config/loader.js";
10
11
  import type { AssistantConfig } from "../../config/schema.js";
11
12
  import * as disabledState from "../../plugins/disabled-state.js";
@@ -245,3 +246,49 @@ describe("createResolveToolsCallback — config.tools.exclude", () => {
245
246
  expect(ctx.allowedToolNames?.has("bash")).toBe(false);
246
247
  });
247
248
  });
249
+
250
+ test("managed browser guidance follows each turn without changing native or shared tool definitions", () => {
251
+ const platform = process.env.IS_PLATFORM;
252
+ const containerized = process.env.IS_CONTAINERIZED;
253
+ process.env.IS_PLATFORM = "true";
254
+ process.env.IS_CONTAINERIZED = "true";
255
+ setOverridesForTesting({ "assistant-desktop": true });
256
+ const bash = def("bash");
257
+ const ctx = makeCtx({
258
+ conversationId: "conv-browser-guidance",
259
+ transportInterface: "web",
260
+ currentTurnClientOs: "web",
261
+ trustContext: { trustClass: "guardian" } as Conversation["trustContext"],
262
+ getTurnActorPrincipalId: () => "user-123",
263
+ });
264
+ const resolve = createResolveToolsCallback([bash], ctx)!;
265
+ try {
266
+ expect(
267
+ resolve([]).find((tool) => tool.name === "bash")?.description,
268
+ ).toContain("assistant browser navigate");
269
+ expect(bash.description).toBe("bash");
270
+ for (const clientOs of ["macos", "windows", "linux"] as const) {
271
+ ctx.currentTurnClientOs = clientOs;
272
+ expect(
273
+ resolve([]).find((tool) => tool.name === "bash")?.description,
274
+ ).toBe("bash");
275
+ }
276
+ ctx.currentTurnClientOs = "web";
277
+ setOverridesForTesting({ "assistant-desktop": false });
278
+ expect(resolve([]).find((tool) => tool.name === "bash")?.description).toBe(
279
+ "bash",
280
+ );
281
+ } finally {
282
+ setOverridesForTesting({});
283
+ if (platform === undefined) {
284
+ delete process.env.IS_PLATFORM;
285
+ } else {
286
+ process.env.IS_PLATFORM = platform;
287
+ }
288
+ if (containerized === undefined) {
289
+ delete process.env.IS_CONTAINERIZED;
290
+ } else {
291
+ process.env.IS_CONTAINERIZED = containerized;
292
+ }
293
+ }
294
+ });
@@ -41,6 +41,7 @@ mock.module("../../mcp/mcp-header-store.js", () => ({
41
41
  }));
42
42
 
43
43
  mock.module("../../config/loader.js", () => ({
44
+ getConfig: () => ({}),
44
45
  invalidateConfigCache: () => {},
45
46
  }));
46
47
 
@@ -0,0 +1,26 @@
1
+ import { type ClientOs, parseClientOs } from "../channels/types.js";
2
+ import type { Conversation } from "./conversation.js";
3
+
4
+ /**
5
+ * Host OS of the client driving this turn. The Electron renderer reports
6
+ * `interface: "web"` and carries the real OS in `clientOs`, so this prefers
7
+ * the frozen per-turn value and only falls back to a desktop transport.
8
+ */
9
+ export function resolveTurnClientOs(ctx: Conversation): {
10
+ clientOs: ClientOs | undefined;
11
+ transportInterface: Conversation["transportInterface"];
12
+ } {
13
+ const pin = ctx.toolContextPin;
14
+ const transportInterface = pin
15
+ ? pin.transportInterface
16
+ : ctx.transportInterface;
17
+ const clientOs = pin
18
+ ? pin.clientOs
19
+ : (parseClientOs(ctx.currentTurnClientOs ?? ctx.clientOs) ??
20
+ (transportInterface === "macos" ||
21
+ transportInterface === "windows" ||
22
+ transportInterface === "linux"
23
+ ? transportInterface
24
+ : undefined));
25
+ return { clientOs, transportInterface };
26
+ }
@@ -854,7 +854,7 @@ export function buildChannelCapabilityBlock(
854
854
  if (clientOs === "macos") {
855
855
  lines.push("");
856
856
  lines.push(
857
- "On macOS, prefer osascript/CLI via `host_bash` over computer use tools, which take over the user's cursor. Use foreground computer use only when no scripting alternative exists or the user explicitly asks.",
857
+ "On macOS, drive apps with the computer-use skill; `host_bash` is for shell commands.",
858
858
  );
859
859
  }
860
860
 
@@ -7,10 +7,9 @@
7
7
  */
8
8
 
9
9
  import type { AssistantEvent } from "../api/index.js";
10
+ import { shouldUseVirtualDesktopBrowser } from "../browser/virtual-desktop-target.js";
10
11
  import {
11
- type ClientOs,
12
12
  type HostProxyCapability,
13
- parseClientOs,
14
13
  supportsHostProxy,
15
14
  } from "../channels/types.js";
16
15
  import { getIsPlatform } from "../config/env-registry.js";
@@ -74,6 +73,7 @@ import {
74
73
  conversationSupportsGuardianQuestionCards,
75
74
  } from "./channel-ui-capability.js";
76
75
  import type { Conversation } from "./conversation.js";
76
+ import { resolveTurnClientOs } from "./conversation-client-surface.js";
77
77
  import { projectSkillTools } from "./conversation-skill-tools.js";
78
78
  import {
79
79
  restoreSurfaceStateEntry,
@@ -742,30 +742,6 @@ export const ALLOWLIST_ONLY_TOOL_NAMES = new Set<string>([
742
742
  "delete_memory_page",
743
743
  ]);
744
744
 
745
- /**
746
- * Host OS of the client driving this turn. The Electron renderer reports
747
- * `interface: "web"` and carries the real OS in `clientOs`, so this prefers
748
- * the frozen per-turn value and only falls back to a desktop transport.
749
- */
750
- function resolveTurnClientOs(ctx: Conversation): {
751
- clientOs: ClientOs | undefined;
752
- transportInterface: Conversation["transportInterface"];
753
- } {
754
- const pin = ctx.toolContextPin;
755
- const transportInterface = pin
756
- ? pin.transportInterface
757
- : ctx.transportInterface;
758
- const clientOs = pin
759
- ? pin.clientOs
760
- : (parseClientOs(ctx.currentTurnClientOs ?? ctx.clientOs) ??
761
- (transportInterface === "macos" ||
762
- transportInterface === "windows" ||
763
- transportInterface === "linux"
764
- ? transportInterface
765
- : undefined));
766
- return { clientOs, transportInterface };
767
- }
768
-
769
745
  /**
770
746
  * Windows parity gate: skill tools may declare `supported_client_os`; drop
771
747
  * them when the turn's client OS (or pinned OS for wakes) is not listed.
@@ -1149,9 +1125,35 @@ export function createResolveToolsCallback(
1149
1125
  // while background calls persist the full surface content for the next
1150
1126
  // capable client that opens the conversation. Skill tools stay off this
1151
1127
  // list (`skill_execute` dispatch) and are a separate disclosure path.
1152
- let allBaseDefs = [...scopedCoreDefs, ...scopedWorkspaceDefs, ...scopedMcpDefs].filter(
1153
- (d) => !excluded.has(d.name),
1154
- );
1128
+ let allBaseDefs = [
1129
+ ...scopedCoreDefs,
1130
+ ...scopedWorkspaceDefs,
1131
+ ...scopedMcpDefs,
1132
+ ].filter((d) => !excluded.has(d.name));
1133
+ if (
1134
+ ctx.transportInterface === "web" &&
1135
+ shouldUseVirtualDesktopBrowser(
1136
+ undefined,
1137
+ {},
1138
+ {
1139
+ workingDir: ctx.workingDir,
1140
+ conversationId: ctx.conversationId,
1141
+ trustClass: ctx.trustContext?.trustClass ?? "unknown",
1142
+ transportInterface: ctx.transportInterface,
1143
+ clientOs: resolveTurnClientOs(ctx).clientOs,
1144
+ sourceActorPrincipalId: ctx.getTurnActorPrincipalId?.(),
1145
+ },
1146
+ )
1147
+ ) {
1148
+ allBaseDefs = allBaseDefs.map((definition) =>
1149
+ definition.name === "bash"
1150
+ ? {
1151
+ ...definition,
1152
+ description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. It installs the virtual desktop if needed, starts Chrome, and completes the action in one call. Use timeout_seconds: ${getConfig().timeouts.shellMaxTimeoutSec} for first use; setup progress is visible in the Virtual desktop panel. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
1153
+ }
1154
+ : definition,
1155
+ );
1156
+ }
1155
1157
  // Activation-rail conversations carry the optional `activation_moment`
1156
1158
  // telemetry param on ui_show. The marker is written before the first
1157
1159
  // tool resolution (see `applyBootstrapTemplate` in system-prompt.ts), so
@@ -48,8 +48,8 @@ const SCREENSHOT_OMITTED_MESSAGE =
48
48
  "Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
49
49
 
50
50
  // computer_use_key combos that change only selection/cursor/clipboard state.
51
- // The AX tree models none of these, so they always produce an empty diff
52
- // exempt them from the "NO VISIBLE EFFECT" signal (mirrors computer_use_wait).
51
+ // The AX tree models none of these, so they always produce an empty diff and
52
+ // are exempt from the unchanged-tree signal.
53
53
  // Stored in canonical form (see canonicalizeKeyCombo): modifier aliases
54
54
  // normalized and ordered, so `cmd + a`, `command+a`, `alt+tab`, `tab+shift`
55
55
  // all match.
@@ -129,12 +129,28 @@ export interface ActionRecord {
129
129
  reasoning?: string;
130
130
  }
131
131
 
132
+ // Steps whose empty diff says nothing about whether they worked: waiting and
133
+ // observing change nothing by design, and an AppleScript reports its own
134
+ // outcome in the execution result, often for an app whose content the tree
135
+ // cannot see. They neither warn nor move the unchanged streak.
136
+ const NO_AX_DIFF_TOOLS = new Set([
137
+ "computer_use_wait",
138
+ "computer_use_observe",
139
+ "computer_use_run_applescript",
140
+ ]);
141
+
142
+ type ActionIdentity = Pick<ActionRecord, "toolName" | "input">;
143
+
144
+ function isNoDiffTool(action: ActionIdentity | undefined): boolean {
145
+ return action !== undefined && NO_AX_DIFF_TOOLS.has(action.toolName);
146
+ }
147
+
132
148
  /**
133
149
  * True when `action` is a computer_use_key press whose key only mutates
134
- * selection/cursor/clipboard state changes the AX tree cannot represent, so
150
+ * selection/cursor/clipboard state: changes the AX tree cannot represent, so
135
151
  * an empty diff is expected rather than a sign the action did nothing.
136
152
  */
137
- function isNoDiffKeyAction(action: ActionRecord | undefined): boolean {
153
+ function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
138
154
  if (action?.toolName !== "computer_use_key") {
139
155
  return false;
140
156
  }
@@ -245,6 +261,7 @@ export class HostCuProxy {
245
261
  resetGeneration: number;
246
262
  dispatchedAt: number;
247
263
  toolName: string;
264
+ input: Record<string, unknown>;
248
265
  step: number;
249
266
  }
250
267
  >();
@@ -455,6 +472,7 @@ export class HostCuProxy {
455
472
  resetGeneration: this._resetGeneration,
456
473
  dispatchedAt: Date.now(),
457
474
  toolName,
475
+ input,
458
476
  step: stepNumber,
459
477
  });
460
478
 
@@ -538,7 +556,7 @@ export class HostCuProxy {
538
556
  }
539
557
 
540
558
  // A targeted snapshot has no comparable action/diff baseline; neither it
541
- // nor the first desktop observation after it can imply "no visible effect".
559
+ // nor the first desktop observation after it can imply an unchanged tree.
542
560
  // A response dispatched before the last reset belongs to a finished run,
543
561
  // so it must not change any state the new run has started building.
544
562
  const fromCurrentRun =
@@ -548,6 +566,11 @@ export class HostCuProxy {
548
566
  this._consecutiveUnchangedSteps = 0;
549
567
  }
550
568
  const prevAXTree = this._previousAXTree;
569
+ // Judge the empty diff against the action this request carried, for the
570
+ // same reason the timings line does.
571
+ const action: ActionIdentity | undefined = owned
572
+ ? { toolName: owned.toolName, input: owned.input }
573
+ : this.lastRecordedAction();
551
574
  const comparableObservation = scopedObservation
552
575
  ? { ...observation, axDiff: undefined, secondaryWindows: undefined }
553
576
  : observation;
@@ -557,7 +580,7 @@ export class HostCuProxy {
557
580
  !scopedObservation &&
558
581
  owned?.toolName !== POINT_AT_PROXY_TOOL
559
582
  ) {
560
- this.updateStateFromObservation(comparableObservation);
583
+ this.updateStateFromObservation(comparableObservation, action);
561
584
  // A desktop has had its first look only once pixels from it arrived,
562
585
  // so a failed capture leaves the next request asking again.
563
586
  if (owned && observation.screenshot) {
@@ -568,6 +591,7 @@ export class HostCuProxy {
568
591
  comparableObservation,
569
592
  prevAXTree,
570
593
  owned?.screenshotSkipped ?? false,
594
+ action,
571
595
  );
572
596
  interaction.rpcResolve(result);
573
597
  return result;
@@ -672,6 +696,7 @@ export class HostCuProxy {
672
696
  obs: CuObservationResult,
673
697
  previousAXTree?: string,
674
698
  screenshotSkipped = false,
699
+ action: ActionIdentity | undefined = this.lastRecordedAction(),
675
700
  ): ToolExecutionResult {
676
701
  const prevTree = previousAXTree;
677
702
  const parts: string[] = [];
@@ -690,24 +715,17 @@ export class HostCuProxy {
690
715
  parts.push(obs.axDiff);
691
716
  parts.push("");
692
717
  } else if (prevTree != null && obs.axTree != null) {
693
- const lastAction =
694
- this._actionHistory.length > 0
695
- ? this._actionHistory[this._actionHistory.length - 1]
696
- : undefined;
697
- const isWaitAction = lastAction?.toolName === "computer_use_wait";
698
- const isNoDiffKey = isNoDiffKeyAction(lastAction);
699
-
700
- if (!isWaitAction && !isNoDiffKey) {
718
+ if (!isNoDiffTool(action) && !isNoDiffKeyAction(action)) {
701
719
  if (
702
720
  this._consecutiveUnchangedSteps >=
703
721
  CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
704
722
  ) {
705
723
  parts.push(
706
- `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
724
+ `WARNING: the accessibility tree did not change across ${this._consecutiveUnchangedSteps} consecutive actions. If the screenshot does not show them working either, try a different approach.`,
707
725
  );
708
726
  } else {
709
727
  parts.push(
710
- "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
728
+ "The accessibility tree did not change after your last action. Apps that draw their own content (timelines, canvases, spreadsheet grids) change without it, so check the screenshot before deciding the action did nothing.",
711
729
  );
712
730
  }
713
731
  parts.push("");
@@ -822,19 +840,26 @@ export class HostCuProxy {
822
840
  // Private helpers
823
841
  // ---------------------------------------------------------------------------
824
842
 
825
- private updateStateFromObservation(obs: CuObservationResult): void {
843
+ private lastRecordedAction(): ActionRecord | undefined {
844
+ return this._actionHistory.at(-1);
845
+ }
846
+
847
+ private updateStateFromObservation(
848
+ obs: CuObservationResult,
849
+ action: ActionIdentity | undefined,
850
+ ): void {
826
851
  if (this._stepCount > 0) {
827
- const lastAction =
828
- this._actionHistory.length > 0
829
- ? this._actionHistory[this._actionHistory.length - 1]
830
- : undefined;
831
- if (obs.axDiff != null || isNoDiffKeyAction(lastAction)) {
852
+ if (obs.axDiff != null || isNoDiffKeyAction(action)) {
832
853
  // A real diff, or an exempt key whose effect is invisible by design,
833
- // breaks the no-effect streak clear it rather than preserving a
834
- // stale count so an intervening cmd+a can't bridge two no-op actions
835
- // into a false "consecutive" escalation.
854
+ // breaks the no-effect streak. Clearing it rather than preserving a
855
+ // stale count keeps an intervening cmd+a from bridging two no-op
856
+ // actions into a false "consecutive" escalation.
836
857
  this._consecutiveUnchangedSteps = 0;
837
- } else if (this._previousAXTree != null && obs.axTree != null) {
858
+ } else if (
859
+ !isNoDiffTool(action) &&
860
+ this._previousAXTree != null &&
861
+ obs.axTree != null
862
+ ) {
838
863
  this._consecutiveUnchangedSteps++;
839
864
  }
840
865
  }
@@ -9,6 +9,8 @@
9
9
  * - `daemon/process-message.ts` (create path, prepareConversationForMessage)
10
10
  * - `daemon/conversation-process.ts` `drainSingleMessage` (re-add after dequeue)
11
11
  * - `daemon/conversation-process.ts` `drainBatch` (re-add after dequeue)
12
+ * - `calls/voice-session-bridge.ts` (tool-capable legs of a live-voice
13
+ * session opened from the macOS desktop client)
12
14
  *
13
15
  * The create paths additionally instantiate the proxy itself; that
14
16
  * instantiation logic is per-proxy-class and stays inline at each create
@@ -14,6 +14,7 @@ import { getMcpServerManager } from "../mcp/manager.js";
14
14
  import { migrateLegacyMcpHeaders } from "../mcp/mcp-header-store.js";
15
15
  import { signalMcpReloaded } from "../mcp/reload-signal.js";
16
16
  import { loadWorkspaceMcpConfig } from "../mcp/workspace-mcp-config.js";
17
+ import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
17
18
  import { createMcpToolsFromServer } from "../tools/mcp/mcp-tool-factory.js";
18
19
  import { registerMcpTools, unregisterAllMcpTools } from "../tools/registry.js";
19
20
  import { getLogger } from "../util/logger.js";
@@ -80,6 +81,7 @@ export async function reconcilePluginMcpServers(): Promise<void> {
80
81
  }
81
82
 
82
83
  async function doReload(): Promise<McpReloadResult> {
84
+ let teardownStarted = false;
83
85
  try {
84
86
  const manager = getMcpServerManager();
85
87
 
@@ -98,6 +100,7 @@ async function doReload(): Promise<McpReloadResult> {
98
100
  invalidateConfigCache();
99
101
 
100
102
  // 2. Stop existing MCP servers + unregister their tools
103
+ teardownStarted = true;
101
104
  await manager.stop();
102
105
  unregisterAllMcpTools();
103
106
 
@@ -160,5 +163,9 @@ async function doReload(): Promise<McpReloadResult> {
160
163
  const error = err instanceof Error ? err.message : String(err);
161
164
  log.error({ err }, "MCP reload failed");
162
165
  return { success: false, error };
166
+ } finally {
167
+ if (teardownStarted) {
168
+ publishMcpChanged();
169
+ }
163
170
  }
164
171
  }
@@ -18,6 +18,7 @@ export const SYNC_TAGS = {
18
18
  appsList: "apps:list",
19
19
  documentsList: "documents:list",
20
20
  pluginsList: "plugins:list",
21
+ mcpList: "mcp:list",
21
22
  conversationsList: "conversations:list",
22
23
  /** Activation-checklist progress: task launches, step counts, completions. */
23
24
  activationProgress: "activation:progress",
@@ -1,6 +1,7 @@
1
1
  import { maybeDefaultSpeechToManaged } from "../config/managed-speech-defaults.js";
2
2
  import { rehydratePlatformCredentials } from "../config/platform-rehydration.js";
3
3
  import type { AssistantConfig } from "../config/types.js";
4
+ import { getMcpServerManager } from "../mcp/manager.js";
4
5
  import { startConfiguredMcpServers } from "../mcp/startup.js";
5
6
  import { gmailMessagingProvider } from "../messaging/providers/gmail/adapter.js";
6
7
  import { outlookMessagingProvider } from "../messaging/providers/outlook/adapter.js";
@@ -9,6 +10,7 @@ import { telegramBotMessagingProvider } from "../messaging/providers/telegram-bo
9
10
  import { whatsappMessagingProvider } from "../messaging/providers/whatsapp/adapter.js";
10
11
  import { registerMessagingProvider } from "../messaging/registry.js";
11
12
  import { initializeProviders } from "../providers/registry.js";
13
+ import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
12
14
  import { validateSubagentRoleAllowlists } from "../subagent/validate-allowlists.js";
13
15
  import { initializeTools } from "../tools/registry.js";
14
16
  import { getLogger } from "../util/logger.js";
@@ -63,7 +65,9 @@ export async function initializeProvidersAndTools(
63
65
  // Start MCP servers — workspace-configured and plugin-declared alike —
64
66
  // and register their tools. Shared with the schedule worker, which hosts
65
67
  // agent turns in its own process and so needs its own connections.
68
+ getMcpServerManager().setUnexpectedCloseHandler(() => publishMcpChanged());
66
69
  await startConfiguredMcpServers();
70
+ publishMcpChanged();
67
71
 
68
72
  log.info("Daemon startup: providers and tools initialized");
69
73
  }
@@ -58,6 +58,7 @@ export function newFakeDesktop(options: FakeDesktopOptions) {
58
58
  let vncReady = true;
59
59
  let chromiumPath: () => Promise<string> = async () => "/fake/chromium";
60
60
  const manager = new DesktopSessionManager({
61
+ allocateDebugPort: async () => 9222,
61
62
  spawn: (role, request) => {
62
63
  if (options.failSpawn?.includes(role)) {
63
64
  throw new Error(`spawn ${role} failed`);