@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/docs/desktop-browser-cli.md +33 -0
  2. package/openapi.yaml +17 -5
  3. package/package.json +1 -1
  4. package/scripts/smoke-desktop-browser-cli.ts +279 -0
  5. package/src/__tests__/config-schema.test.ts +1 -1
  6. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  7. package/src/__tests__/conversation-error.test.ts +4 -1
  8. package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
  9. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  10. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  11. package/src/__tests__/host-cu-proxy.test.ts +116 -18
  12. package/src/__tests__/list-all-apps.test.ts +40 -8
  13. package/src/__tests__/mcp-auth-routes.test.ts +99 -0
  14. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  15. package/src/__tests__/mcp-health-check.test.ts +32 -32
  16. package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
  17. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  18. package/src/__tests__/skills.test.ts +10 -0
  19. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  20. package/src/acp/session-snapshot.ts +260 -0
  21. package/src/apps/app-store.ts +6 -5
  22. package/src/browser/operations.ts +2 -1
  23. package/src/browser/types.ts +7 -0
  24. package/src/browser/virtual-desktop-target.ts +34 -0
  25. package/src/calls/__tests__/progress-narration.test.ts +23 -0
  26. package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
  27. package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
  28. package/src/calls/progress-narration.ts +15 -2
  29. package/src/calls/voice-control-protocol.ts +34 -4
  30. package/src/calls/voice-session-bridge.ts +23 -0
  31. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  32. package/src/cli/commands/browser.help.ts +43 -0
  33. package/src/cli/commands/browser.ts +52 -8
  34. package/src/cli/commands/plugins.ts +16 -5
  35. package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
  36. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  37. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  38. package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
  39. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  40. package/src/cli/lib/bundled-marketplace.json +13 -0
  41. package/src/cli/lib/install-from-github.ts +17 -18
  42. package/src/cli/lib/install-from-platform.ts +14 -9
  43. package/src/cli/lib/list-installed-plugins.ts +21 -35
  44. package/src/cli/lib/plugin-details.ts +21 -8
  45. package/src/cli/lib/uninstall-plugin.ts +59 -1
  46. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  47. package/src/config/bundled-skills/acp/SKILL.md +1 -1
  48. package/src/config/bundled-skills/acp/TOOLS.json +2 -2
  49. package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
  50. package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
  51. package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
  52. package/src/config/feature-flag-registry.json +2 -2
  53. package/src/config/schemas/__tests__/voice.test.ts +2 -2
  54. package/src/config/schemas/mcp.ts +9 -7
  55. package/src/config/schemas/monitoring.ts +9 -0
  56. package/src/config/schemas/voice.ts +2 -2
  57. package/src/config/skills.ts +13 -21
  58. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  59. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
  60. package/src/daemon/conversation-client-surface.ts +26 -0
  61. package/src/daemon/conversation-runtime-assembly.ts +1 -1
  62. package/src/daemon/conversation-tool-setup.ts +31 -29
  63. package/src/daemon/host-cu-proxy.ts +51 -26
  64. package/src/daemon/host-proxy-preactivation.ts +2 -0
  65. package/src/daemon/mcp-reload-service.ts +7 -0
  66. package/src/daemon/message-types/sync.ts +1 -0
  67. package/src/daemon/providers-setup.ts +4 -0
  68. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  69. package/src/desktop/desktop-automation-lease.test.ts +208 -0
  70. package/src/desktop/desktop-automation-lease.ts +270 -0
  71. package/src/desktop/desktop-browser-client.test.ts +413 -0
  72. package/src/desktop/desktop-browser-client.ts +430 -0
  73. package/src/desktop/desktop-browser-cursor.ts +40 -0
  74. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  75. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  76. package/src/desktop/desktop-browser-operations.ts +130 -0
  77. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  78. package/src/desktop/desktop-chrome-session.ts +58 -8
  79. package/src/desktop/desktop-dependencies.test.ts +29 -0
  80. package/src/desktop/desktop-dependencies.ts +26 -5
  81. package/src/desktop/desktop-display.ts +9 -0
  82. package/src/desktop/desktop-panel-config.ts +14 -1
  83. package/src/desktop/desktop-session-manager.test.ts +86 -3
  84. package/src/desktop/desktop-session-manager.ts +144 -24
  85. package/src/desktop/desktop-stream-bridge.ts +1 -1
  86. package/src/desktop/virtual-desktop-feature.ts +18 -0
  87. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  88. package/src/i18n/__tests__/i18n.test.ts +9 -5
  89. package/src/i18n/messages.ts +13 -4
  90. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  91. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  92. package/src/ipc/assistant-server.ts +15 -1
  93. package/src/ipc/cli-client.ts +13 -5
  94. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
  95. package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
  96. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  97. package/src/live-voice/__tests__/protocol.test.ts +11 -3
  98. package/src/live-voice/__tests__/session-controls.test.ts +34 -0
  99. package/src/live-voice/live-voice-metrics.ts +9 -0
  100. package/src/live-voice/live-voice-session.ts +3 -0
  101. package/src/live-voice/protocol.ts +10 -1
  102. package/src/live-voice/session-controls.ts +28 -0
  103. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  104. package/src/mcp/__tests__/effective-config.test.ts +11 -0
  105. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  106. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  107. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  108. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  109. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  110. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  111. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  112. package/src/mcp/client.ts +64 -16
  113. package/src/mcp/credential-target.ts +105 -0
  114. package/src/mcp/effective-config.ts +42 -15
  115. package/src/mcp/manager.ts +61 -8
  116. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  117. package/src/mcp/mcp-auth-state.ts +38 -6
  118. package/src/mcp/mcp-oauth-provider.ts +101 -33
  119. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  120. package/src/monitoring/mount-watch.ts +348 -0
  121. package/src/monitoring/worker.ts +9 -0
  122. package/src/notifications/__tests__/decision-engine.test.ts +175 -0
  123. package/src/notifications/decision-engine.ts +43 -8
  124. package/src/oauth/seed-providers.ts +11 -0
  125. package/src/onboarding/checkin-event.ts +2 -1
  126. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  127. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  128. package/src/plugins/external-plugin-loader.ts +49 -86
  129. package/src/plugins/installed-plugin-dirs.ts +6 -4
  130. package/src/plugins/mcp-servers.ts +9 -15
  131. package/src/plugins/mtime-cache.ts +2 -1
  132. package/src/plugins/user-loader.ts +2 -1
  133. package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
  134. package/src/providers/inference/endpoint-probe.ts +27 -12
  135. package/src/providers/openai/responses-provider.ts +9 -2
  136. package/src/providers/opencode/client.test.ts +167 -0
  137. package/src/providers/opencode/client.ts +73 -4
  138. package/src/runtime/AGENTS.md +4 -0
  139. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
  140. package/src/runtime/http-server.ts +6 -6
  141. package/src/runtime/routes/__tests__/acp-routes.test.ts +19 -0
  142. package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
  143. package/src/runtime/routes/acp-routes.ts +17 -161
  144. package/src/runtime/routes/browser-context.ts +39 -0
  145. package/src/runtime/routes/browser-routes.ts +16 -47
  146. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  147. package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
  148. package/src/runtime/routes/desktop-setup-routes.ts +7 -5
  149. package/src/runtime/routes/mcp-auth-routes.ts +106 -94
  150. package/src/runtime/routes/plugins-routes.ts +14 -1
  151. package/src/runtime/sync/resource-sync-events.ts +4 -0
  152. package/src/tools/acp/status.test.ts +276 -21
  153. package/src/tools/acp/status.ts +98 -32
  154. package/src/tools/browser/browser-execution.ts +44 -13
  155. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  156. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  157. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  158. package/src/tools/computer-use/definitions.ts +2 -3
  159. package/src/util/abort-reasons.ts +12 -1
  160. package/src/util/plugin-manifest.ts +203 -0
  161. package/src/desktop/desktop-feature.test.ts +0 -18
  162. package/src/desktop/desktop-feature.ts +0 -19
@@ -1,13 +1,37 @@
1
1
  import { z } from "zod";
2
2
 
3
- import { getAcpSessionManager } from "../../acp/index.js";
4
- import type { AcpSessionState } from "../../acp/types.js";
3
+ import { resolvedClaudeCredentialDigest } from "../../acp/prepare-agent-env.js";
4
+ import {
5
+ type AcpSessionSnapshot,
6
+ getAcpSessionSnapshot,
7
+ listAcpSessionSnapshots,
8
+ withCurrentAuthMarkers,
9
+ } from "../../acp/session-snapshot.js";
5
10
  import {
6
11
  invalidToolInputResult,
7
12
  nullAsOmitted,
8
13
  } from "../shared/zod-tool-schema.js";
9
14
  import type { ToolContext, ToolExecutionResult } from "../types.js";
10
15
 
16
+ const STATUS_LIST_LIMIT = 50;
17
+
18
+ /**
19
+ * Keep every live session visible, then fill remaining capacity from durable
20
+ * history. An older active run must not be displaced by newer historical rows.
21
+ */
22
+ function selectStatusSnapshots(
23
+ sessions: AcpSessionSnapshot[],
24
+ ): AcpSessionSnapshot[] {
25
+ const live = sessions.filter((session) => session.source === "live");
26
+ const history = sessions
27
+ .filter((session) => session.source === "history")
28
+ .sort((a, b) => b.startedAt - a.startedAt);
29
+ const remaining = Math.max(0, STATUS_LIST_LIMIT - live.length);
30
+ return [...live, ...history.slice(0, remaining)].sort(
31
+ (a, b) => b.startedAt - a.startedAt,
32
+ );
33
+ }
34
+
11
35
  /**
12
36
  * Model-input schema, `safeParse`d at the top of {@link executeAcpStatus}.
13
37
  * Same in-tool pattern and TOOLS.json drift guard as the other bundled-skill
@@ -24,33 +48,48 @@ export const acpStatusInputSchema = z.looseObject({
24
48
  * `AcpSessionState` for those surfaces should not reach this one by default.
25
49
  * `model` stays: which model a session is running on is answerable.
26
50
  */
27
- function projectSession(state: AcpSessionState) {
51
+ function projectSession(snapshot: AcpSessionSnapshot) {
52
+ const fromHistory = snapshot.source === "history";
53
+ const idle =
54
+ fromHistory &&
55
+ snapshot.status === "completed" &&
56
+ snapshot.stopReason !== "cancelled" &&
57
+ snapshot.resumable;
58
+ const latestUsage =
59
+ snapshot.usedTokens !== undefined && snapshot.contextSize !== undefined
60
+ ? {
61
+ usedTokens: snapshot.usedTokens,
62
+ contextSize: snapshot.contextSize,
63
+ costAmount: snapshot.costAmount,
64
+ costCurrency: snapshot.costCurrency,
65
+ inputTokens: snapshot.inputTokens,
66
+ outputTokens: snapshot.outputTokens,
67
+ }
68
+ : undefined;
28
69
  return {
29
- id: state.id,
30
- agentId: state.agentId,
31
- acpSessionId: state.acpSessionId,
32
- parentConversationId: state.parentConversationId,
33
- status: state.status,
34
- startedAt: state.startedAt,
35
- completedAt: state.completedAt,
36
- error: state.error,
37
- stopReason: state.stopReason,
38
- task: state.task,
39
- parentToolUseId: state.parentToolUseId,
40
- authErrorCode: state.authErrorCode,
41
- authErrorCredential: state.authErrorCredential,
42
- latestUsage: state.latestUsage,
43
- model: state.model,
70
+ id: snapshot.id,
71
+ agentId: snapshot.agentId,
72
+ acpSessionId: snapshot.acpSessionId,
73
+ parentConversationId: snapshot.parentConversationId,
74
+ status: idle ? "idle" : snapshot.status,
75
+ startedAt: snapshot.startedAt,
76
+ completedAt: snapshot.completedAt ?? undefined,
77
+ error: snapshot.error ?? undefined,
78
+ stopReason: snapshot.stopReason ?? undefined,
79
+ task: snapshot.task,
80
+ parentToolUseId: snapshot.parentToolUseId,
81
+ authErrorCode: snapshot.authErrorCode,
82
+ latestUsage,
83
+ model: snapshot.model,
84
+ ...(fromHistory
85
+ ? {
86
+ resumable: snapshot.resumable,
87
+ lastRunStatus: snapshot.status,
88
+ }
89
+ : {}),
44
90
  };
45
91
  }
46
92
 
47
- /** Projects either shape `getStatus` answers with. */
48
- function projectStatus(status: AcpSessionState | AcpSessionState[]): unknown {
49
- return Array.isArray(status)
50
- ? status.map(projectSession)
51
- : projectSession(status);
52
- }
53
-
54
93
  export async function executeAcpStatus(
55
94
  input: Record<string, unknown>,
56
95
  _context: ToolContext,
@@ -60,24 +99,51 @@ export async function executeAcpStatus(
60
99
  return invalidToolInputResult("acp_status", parsedInput.error);
61
100
  }
62
101
  const acpSessionId = parsedInput.data.acp_session_id;
63
- const manager = getAcpSessionManager();
64
-
65
102
  try {
66
103
  if (acpSessionId) {
104
+ const snapshot = getAcpSessionSnapshot(acpSessionId, {
105
+ includeEventLog: false,
106
+ });
107
+ if (!snapshot) {
108
+ return {
109
+ content: `ACP session "${acpSessionId}" not found`,
110
+ isError: true,
111
+ };
112
+ }
113
+ const judged = (
114
+ await withCurrentAuthMarkers(
115
+ [snapshot],
116
+ resolvedClaudeCredentialDigest,
117
+ )
118
+ )[0];
119
+ if (!judged) {
120
+ return {
121
+ content: `ACP session "${acpSessionId}" not found`,
122
+ isError: true,
123
+ };
124
+ }
67
125
  return {
68
- content: JSON.stringify(projectStatus(manager.getStatus(acpSessionId))),
126
+ content: JSON.stringify(projectSession(judged)),
69
127
  isError: false,
70
128
  };
71
129
  }
72
130
 
73
- // List all sessions.
74
- const allStates = manager.getStatus();
75
- if (Array.isArray(allStates) && allStates.length === 0) {
131
+ const snapshots = selectStatusSnapshots(
132
+ listAcpSessionSnapshots({
133
+ limit: STATUS_LIST_LIMIT,
134
+ includeEventLog: false,
135
+ }).sessions,
136
+ );
137
+ if (snapshots.length === 0) {
76
138
  return { content: "No ACP sessions found.", isError: false };
77
139
  }
78
140
 
141
+ const judged = await withCurrentAuthMarkers(
142
+ snapshots,
143
+ resolvedClaudeCredentialDigest,
144
+ );
79
145
  return {
80
- content: JSON.stringify(projectStatus(allStates)),
146
+ content: JSON.stringify(judged.map(projectSession)),
81
147
  isError: false,
82
148
  };
83
149
  } catch (err) {
@@ -1,4 +1,5 @@
1
1
  import { optimizeImageForTransport } from "../../agent/image-optimize.js";
2
+ import type { BrowserOperationContext as ToolContext } from "../../browser/types.js";
2
3
  import { getConfig } from "../../config/loader.js";
3
4
  import { HostBrowserProxy } from "../../daemon/host-browser-proxy.js";
4
5
  import type { ImageContent } from "../../providers/types.js";
@@ -21,7 +22,7 @@ import {
21
22
  sanitizeUrlForOutput,
22
23
  sanitizeUrlStringForOutput,
23
24
  } from "../network/url-safety.js";
24
- import type { ToolContext, ToolExecutionResult } from "../types.js";
25
+ import type { ToolExecutionResult } from "../types.js";
25
26
  import {
26
27
  type AuthChallenge,
27
28
  detectAuthChallenge,
@@ -515,6 +516,9 @@ async function acquireCdpClientWithMode(
515
516
  }
516
517
  | { cdp?: never; browserMode?: never; errorResult: ToolExecutionResult }
517
518
  > {
519
+ if (context.cdpClient) {
520
+ return { cdp: context.cdpClient, browserMode: "cdp-inspect" };
521
+ }
518
522
  const modeResult = parseBrowserMode(input);
519
523
  if (!modeResult.ok) {
520
524
  return {
@@ -746,6 +750,17 @@ export type ResolvedElement =
746
750
  | { kind: "backend"; backendNodeId: number; eid: string }
747
751
  | { kind: "selector"; selector: string };
748
752
 
753
+ function snapshotConversationId(context: ToolContext): string {
754
+ return context.cdpClient?.conversationId ?? context.conversationId;
755
+ }
756
+
757
+ function clearBrowserSessionState(context: ToolContext): void {
758
+ browserManager.clearSnapshotBackendNodeMap(snapshotConversationId(context));
759
+ if (!context.cdpClient) {
760
+ browserManager.clearPreferredBackendKind(context.conversationId);
761
+ }
762
+ }
763
+
749
764
  /**
750
765
  * Resolve an element reference (either `element_id` from a prior
751
766
  * snapshot or a raw `selector`) for CDP-native tools. Returns a
@@ -872,6 +887,9 @@ export async function executeBrowserNavigate(
872
887
  typeof input.target_client_id === "string" && input.target_client_id !== ""
873
888
  ? input.target_client_id
874
889
  : undefined;
890
+ if (context.cdpClient && forceNewTab) {
891
+ await cdp.send("Vellum.createTab", {}, context.signal);
892
+ }
875
893
  if (cdp.kind === "extension" && useActiveTab) {
876
894
  // Explicit opt-out: target the currently-active tab. Clear any
877
895
  // conversation pin and reset the live session so this navigate is
@@ -1188,7 +1206,7 @@ export async function executeBrowserNavigate(
1188
1206
  // Navigation changed the page content, so clear stale snapshot
1189
1207
  // mappings regardless of backend. The backendNodeId map is shared
1190
1208
  // per-conversation state that needs to be invalidated on any nav.
1191
- browserManager.clearSnapshotBackendNodeMap(context.conversationId);
1209
+ browserManager.clearSnapshotBackendNodeMap(snapshotConversationId(context));
1192
1210
 
1193
1211
  // Auto-dismiss common blocker modals (regulatory notices, cookie
1194
1212
  // banners) that aren't exposed in the accessibility tree. Runs
@@ -1398,7 +1416,7 @@ export async function executeBrowserSnapshot(
1398
1416
  const { elements, selectorMap: backendNodeMap } = transformAxTree(rawTree);
1399
1417
 
1400
1418
  browserManager.storeSnapshotBackendNodeMap(
1401
- context.conversationId,
1419
+ snapshotConversationId(context),
1402
1420
  backendNodeMap,
1403
1421
  );
1404
1422
 
@@ -1600,8 +1618,7 @@ export async function executeBrowserDetach(
1600
1618
  // Vellum.detach round-trip failed (target gone, transport dropped).
1601
1619
  // browser_detach is the user's recovery path — leaving a stale
1602
1620
  // sticky backend or snapshot map behind would defeat its purpose.
1603
- browserManager.clearSnapshotBackendNodeMap(context.conversationId);
1604
- browserManager.clearPreferredBackendKind(context.conversationId);
1621
+ clearBrowserSessionState(context);
1605
1622
  cdp.dispose();
1606
1623
  }
1607
1624
  }
@@ -1653,8 +1670,7 @@ export async function executeBrowserClose(
1653
1670
  // Tolerate detach failures (already detached, tab closed, etc.)
1654
1671
  }
1655
1672
  }
1656
- browserManager.clearSnapshotBackendNodeMap(context.conversationId);
1657
- browserManager.clearPreferredBackendKind(context.conversationId);
1673
+ clearBrowserSessionState(context);
1658
1674
  return {
1659
1675
  content:
1660
1676
  "Browser session cleared. (Your Chrome tab was not closed — close it yourself if desired.)",
@@ -1683,7 +1699,10 @@ export async function executeBrowserClick(
1683
1699
  input: Record<string, unknown>,
1684
1700
  context: ToolContext,
1685
1701
  ): Promise<ToolExecutionResult> {
1686
- const { resolved, error } = resolveElement(context.conversationId, input);
1702
+ const { resolved, error } = resolveElement(
1703
+ snapshotConversationId(context),
1704
+ input,
1705
+ );
1687
1706
  if (error) {
1688
1707
  return { content: error, isError: true };
1689
1708
  }
@@ -1796,7 +1815,10 @@ export async function executeBrowserType(
1796
1815
  input: Record<string, unknown>,
1797
1816
  context: ToolContext,
1798
1817
  ): Promise<ToolExecutionResult> {
1799
- const { resolved, error } = resolveElement(context.conversationId, input);
1818
+ const { resolved, error } = resolveElement(
1819
+ snapshotConversationId(context),
1820
+ input,
1821
+ );
1800
1822
  if (error) {
1801
1823
  return { content: error, isError: true };
1802
1824
  }
@@ -1887,7 +1909,7 @@ export async function executeBrowserPressKey(
1887
1909
  let targetDescription: string | null = null;
1888
1910
  let resolved: ResolvedElement | null = null;
1889
1911
  if (hasTarget) {
1890
- const res = resolveElement(context.conversationId, input);
1912
+ const res = resolveElement(snapshotConversationId(context), input);
1891
1913
  if (res.error) {
1892
1914
  return { content: res.error, isError: true };
1893
1915
  }
@@ -2025,7 +2047,10 @@ export async function executeBrowserSelectOption(
2025
2047
  input: Record<string, unknown>,
2026
2048
  context: ToolContext,
2027
2049
  ): Promise<ToolExecutionResult> {
2028
- const { resolved, error } = resolveElement(context.conversationId, input);
2050
+ const { resolved, error } = resolveElement(
2051
+ snapshotConversationId(context),
2052
+ input,
2053
+ );
2029
2054
  if (error) {
2030
2055
  return { content: error, isError: true };
2031
2056
  }
@@ -2162,7 +2187,10 @@ export async function executeBrowserHover(
2162
2187
  input: Record<string, unknown>,
2163
2188
  context: ToolContext,
2164
2189
  ): Promise<ToolExecutionResult> {
2165
- const { resolved, error } = resolveElement(context.conversationId, input);
2190
+ const { resolved, error } = resolveElement(
2191
+ snapshotConversationId(context),
2192
+ input,
2193
+ );
2166
2194
  if (error) {
2167
2195
  return { content: error, isError: true };
2168
2196
  }
@@ -2410,7 +2438,10 @@ export async function executeBrowserFillCredential(
2410
2438
  return { content: "Error: field is required.", isError: true };
2411
2439
  }
2412
2440
 
2413
- const { resolved, error } = resolveElement(context.conversationId, input);
2441
+ const { resolved, error } = resolveElement(
2442
+ snapshotConversationId(context),
2443
+ input,
2444
+ );
2414
2445
  if (error) {
2415
2446
  return { content: error, isError: true };
2416
2447
  }
@@ -2804,3 +2804,72 @@ describe("macOS host-browser proxy without extension registry", () => {
2804
2804
  expect(result).toEqual({ ok: true, via: "extension" });
2805
2805
  });
2806
2806
  });
2807
+
2808
+ describe("native renderer browser compatibility", () => {
2809
+ beforeEach(() => {
2810
+ setCdpInspectEnabled(false);
2811
+ setDesktopAutoConfig({ enabled: true, cooldownMs: 30_000 });
2812
+ _resetDesktopAutoCooldown();
2813
+ _resetHostBridgeCooldown();
2814
+ mockSingletonProxy = null;
2815
+ });
2816
+ test.each(["macos", "windows"] as const)(
2817
+ "%s web renderer sends CDP to the connected host bridge",
2818
+ async (clientOs) => {
2819
+ mockSingletonProxy = makeMacosBridgeOnlyProxy();
2820
+ const client = getCdpClient(
2821
+ makeContext({
2822
+ conversationId: "conv-native",
2823
+ transportInterface: "web",
2824
+ clientOs,
2825
+ }),
2826
+ );
2827
+ try {
2828
+ expect(await client.send("Page.getFrameTree", {})).toMatchObject({
2829
+ via: "host-bridge",
2830
+ });
2831
+ } finally {
2832
+ client.dispose();
2833
+ }
2834
+ },
2835
+ );
2836
+
2837
+ test.each(["macos", "windows"] as const)(
2838
+ "%s web renderer keeps the existing Playwright fallback without a bridge",
2839
+ async (clientOs) => {
2840
+ const client = getCdpClient(
2841
+ makeContext({
2842
+ conversationId: "conv-native",
2843
+ transportInterface: "web",
2844
+ clientOs,
2845
+ }),
2846
+ );
2847
+ try {
2848
+ expect(await client.send("Page.getFrameTree", {})).toMatchObject({
2849
+ via: "local",
2850
+ });
2851
+ } finally {
2852
+ client.dispose();
2853
+ }
2854
+ },
2855
+ );
2856
+
2857
+ test("explicit Playwright choice overrides the native browser default", async () => {
2858
+ mockSingletonProxy = makeMacosBridgeOnlyProxy();
2859
+ const client = getCdpClient(
2860
+ makeContext({
2861
+ conversationId: "conv-native",
2862
+ transportInterface: "web",
2863
+ clientOs: "windows",
2864
+ }),
2865
+ { mode: "local" },
2866
+ );
2867
+ try {
2868
+ expect(await client.send("Page.getFrameTree", {})).toMatchObject({
2869
+ via: "local",
2870
+ });
2871
+ } finally {
2872
+ client.dispose();
2873
+ }
2874
+ });
2875
+ });
@@ -417,6 +417,7 @@ describe("connectCdpWsTransport", () => {
417
417
  }
418
418
  expect(caught).toBeInstanceOf(CdpWsTransportError);
419
419
  expect((caught as CdpWsTransportError).code).toBe("closed");
420
+ expect(transport.closed).toBe(true);
420
421
 
421
422
  // After close, subsequent sends also reject with closed.
422
423
  let caught2: unknown;
@@ -99,6 +99,8 @@ export interface CdpTransportEvent {
99
99
  * CDP error mapping to the shared `CdpError` taxonomy.
100
100
  */
101
101
  export interface CdpWsTransport {
102
+ readonly closed?: boolean;
103
+
102
104
  /**
103
105
  * Send a CDP method call over the socket and await its response.
104
106
  *
@@ -464,6 +466,9 @@ function createTransport(ws: WsLike): CdpWsTransport {
464
466
  ws.addEventListener("error", handleError);
465
467
 
466
468
  const transport: CdpWsTransport = {
469
+ get closed() {
470
+ return disposed || closed;
471
+ },
467
472
  send<T = unknown>(
468
473
  method: string,
469
474
  params?: Record<string, unknown>,
@@ -352,7 +352,7 @@ export const computerUseOpenAppTool = {
352
352
  export const computerUseRunAppleScriptTool = {
353
353
  name: "computer_use_run_applescript",
354
354
  description:
355
- "Run an AppleScript command. Prefer this over click/type when possible - it doesn't move the cursor or interrupt foreground activity. Never use 'do shell script' inside AppleScript (blocked for security).",
355
+ "Run an AppleScript on the Mac. Try the target app's own scripting dictionary first, asking it for the state you want rather than for the clicks that would produce it; fall back to System Events menu clicking for apps with no dictionary entry for what you need. Prefer this over click and type when the app supports it; it does not move the cursor. The result is the script's return value, and the accessibility tree may not change even when the script worked. Never use 'do shell script' inside AppleScript (blocked for security).",
356
356
  category: "computer-use",
357
357
  defaultRiskLevel: RiskLevel.Low,
358
358
  executionTarget: "host",
@@ -367,8 +367,7 @@ export const computerUseRunAppleScriptTool = {
367
367
  },
368
368
  reasoning: {
369
369
  type: "string",
370
- description:
371
- "Explanation of what this script does and why AppleScript is better than UI interaction for this step",
370
+ description: "What this script does",
372
371
  },
373
372
  target_client_id: {
374
373
  type: "string",
@@ -21,7 +21,18 @@ export type AbortReasonKind =
21
21
  /** A signal-file cancel was written by an out-of-process caller (CLI, hook). */
22
22
  | "signal_cancel"
23
23
  /** Voice session bridge aborted the conversation (turn supersession, call end). */
24
- | "voice_session_aborted";
24
+ | "voice_session_aborted"
25
+ /**
26
+ * One spoken progress update ran out of its generation budget.
27
+ *
28
+ * Distinct from {@link "voice_session_aborted"} because nothing was aborted
29
+ * but the update itself: the call is still up, the turn is still running,
30
+ * and the only consequence is that this beat goes unspoken. Sharing the
31
+ * session-abort kind made a narrator that had gone fail-open read in the
32
+ * logs as a session dying over and over, which is the opposite of what the
33
+ * line means.
34
+ */
35
+ | "voice_progress_narration_timeout";
25
36
 
26
37
  const ABORT_REASON_TAG = "__vellumAbortReason" as const;
27
38
 
@@ -0,0 +1,203 @@
1
+ /**
2
+ * Resolve the manifest that identifies an installed plugin.
3
+ *
4
+ * `package.json` is the primary manifest for Vellum plugins. A root
5
+ * `plugin.json` is selected only when `package.json` is absent, so a legacy
6
+ * plugin can carry another ecosystem's `plugin.json` without being
7
+ * reclassified.
8
+ */
9
+
10
+ import { existsSync, readFileSync } from "node:fs";
11
+ import { join } from "node:path";
12
+
13
+ import { z } from "zod";
14
+
15
+ export const LEGACY_PLUGIN_MANIFEST = "package.json";
16
+ export const STANDARD_PLUGIN_MANIFEST = "plugin.json";
17
+ export const AGENT_PLUGINS_MANIFEST_SCHEMA_URL =
18
+ "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json";
19
+
20
+ const PluginNameSchema = z
21
+ .string()
22
+ .min(1)
23
+ .max(64)
24
+ .regex(/^[a-z0-9](?!.*(?:--|\.\.))[a-z0-9.-]*[a-z0-9]$|^[a-z0-9]$/);
25
+
26
+ const LegacyPluginManifestSchema = z
27
+ .object({
28
+ name: z.string().min(1),
29
+ version: z.string().optional(),
30
+ description: z.string().min(1).optional(),
31
+ homepage: z.string().optional(),
32
+ license: z.unknown().optional(),
33
+ peerDependencies: z.record(z.string(), z.string()).optional(),
34
+ credentialKeyPatterns: z.unknown().optional(),
35
+ displayName: z.string().min(1).optional(),
36
+ icon: z.string().min(1).optional(),
37
+ })
38
+ .passthrough();
39
+
40
+ const StandardPluginManifestSchema = z
41
+ .object({
42
+ $schema: z.literal(AGENT_PLUGINS_MANIFEST_SCHEMA_URL),
43
+ name: PluginNameSchema,
44
+ version: z.string().optional(),
45
+ description: z.string().optional(),
46
+ author: z
47
+ .object({
48
+ name: z.string().optional(),
49
+ email: z.string().optional(),
50
+ url: z.string().optional(),
51
+ })
52
+ .strict()
53
+ .optional(),
54
+ homepage: z.string().optional(),
55
+ repository: z.string().optional(),
56
+ license: z.string().optional(),
57
+ keywords: z.array(z.string()).optional(),
58
+ // The core specification says a non-object extensions value is reported
59
+ // and ignored rather than rejecting the plugin. This host does not consume
60
+ // a standard extension namespace here, so keep it opaque.
61
+ extensions: z.unknown().optional(),
62
+ })
63
+ // Unknown core fields are reported-and-ignored by the specification. The
64
+ // loader does not assign them semantics, so passthrough preserves that
65
+ // non-fatal behavior without copying them into runtime metadata.
66
+ .passthrough();
67
+
68
+ export type PluginManifestSource =
69
+ | typeof LEGACY_PLUGIN_MANIFEST
70
+ | typeof STANDARD_PLUGIN_MANIFEST;
71
+
72
+ export interface ResolvedPluginManifest {
73
+ readonly source: PluginManifestSource;
74
+ readonly name: string;
75
+ readonly version?: string;
76
+ readonly description?: string;
77
+ readonly homepage?: string;
78
+ readonly license?: unknown;
79
+ /** Parsed selected manifest, retained for Vellum-specific legacy fields. */
80
+ readonly raw: Record<string, unknown>;
81
+ }
82
+
83
+ export type PluginManifestInstallAction = "keep" | "synthesize-legacy";
84
+
85
+ export class PluginManifestError extends Error {
86
+ constructor(
87
+ readonly pluginDir: string,
88
+ readonly source: PluginManifestSource | null,
89
+ message: string,
90
+ readonly cause?: unknown,
91
+ ) {
92
+ super(message);
93
+ this.name = "PluginManifestError";
94
+ }
95
+ }
96
+
97
+ /** True when the directory carries either supported root manifest. */
98
+ export function hasPluginManifest(pluginDir: string): boolean {
99
+ return (
100
+ existsSync(join(pluginDir, LEGACY_PLUGIN_MANIFEST)) ||
101
+ existsSync(join(pluginDir, STANDARD_PLUGIN_MANIFEST))
102
+ );
103
+ }
104
+
105
+ /**
106
+ * Decide whether an installed artifact needs the compatibility `package.json`.
107
+ *
108
+ * A `package.json` file is always preserved. A root `plugin.json`
109
+ * claims the Agent Plugins format only when it names the supported schema. A
110
+ * claimed standard manifest is validated before install; other `plugin.json`
111
+ * files remain foreign metadata beside the synthesized compatibility manifest.
112
+ */
113
+ export function getPluginManifestInstallAction(
114
+ pluginDir: string,
115
+ ): PluginManifestInstallAction {
116
+ if (existsSync(join(pluginDir, LEGACY_PLUGIN_MANIFEST))) {
117
+ return "keep";
118
+ }
119
+
120
+ const standardPath = join(pluginDir, STANDARD_PLUGIN_MANIFEST);
121
+ if (!existsSync(standardPath)) {
122
+ return "synthesize-legacy";
123
+ }
124
+
125
+ let json: unknown;
126
+ try {
127
+ json = JSON.parse(readFileSync(standardPath, "utf8"));
128
+ } catch {
129
+ return "synthesize-legacy";
130
+ }
131
+
132
+ if (
133
+ typeof json !== "object" ||
134
+ json === null ||
135
+ Array.isArray(json) ||
136
+ (json as Record<string, unknown>).$schema !==
137
+ AGENT_PLUGINS_MANIFEST_SCHEMA_URL
138
+ ) {
139
+ return "synthesize-legacy";
140
+ }
141
+
142
+ readPluginManifest(pluginDir);
143
+ return "keep";
144
+ }
145
+
146
+ /**
147
+ * Read and validate the selected root manifest.
148
+ *
149
+ * Selection is deterministic: `package.json` wins when present, including
150
+ * when it is malformed. The loader never falls through
151
+ * from a selected malformed legacy manifest to `plugin.json`.
152
+ */
153
+ export function readPluginManifest(pluginDir: string): ResolvedPluginManifest {
154
+ const source = existsSync(join(pluginDir, LEGACY_PLUGIN_MANIFEST))
155
+ ? LEGACY_PLUGIN_MANIFEST
156
+ : existsSync(join(pluginDir, STANDARD_PLUGIN_MANIFEST))
157
+ ? STANDARD_PLUGIN_MANIFEST
158
+ : null;
159
+ if (source === null) {
160
+ throw new PluginManifestError(
161
+ pluginDir,
162
+ null,
163
+ `plugin at ${pluginDir} is missing package.json and plugin.json`,
164
+ );
165
+ }
166
+
167
+ const path = join(pluginDir, source);
168
+ let json: unknown;
169
+ try {
170
+ json = JSON.parse(readFileSync(path, "utf8"));
171
+ } catch (cause) {
172
+ const reason = cause instanceof Error ? cause.message : String(cause);
173
+ throw new PluginManifestError(
174
+ pluginDir,
175
+ source,
176
+ `${source} at ${pluginDir} could not be read or parsed: ${reason}`,
177
+ cause,
178
+ );
179
+ }
180
+
181
+ const parsed =
182
+ source === LEGACY_PLUGIN_MANIFEST
183
+ ? LegacyPluginManifestSchema.safeParse(json)
184
+ : StandardPluginManifestSchema.safeParse(json);
185
+ if (!parsed.success) {
186
+ throw new PluginManifestError(
187
+ pluginDir,
188
+ source,
189
+ `${source} at ${pluginDir} failed schema validation: ${parsed.error.message}`,
190
+ parsed.error,
191
+ );
192
+ }
193
+
194
+ return {
195
+ source,
196
+ name: parsed.data.name,
197
+ version: parsed.data.version,
198
+ description: parsed.data.description,
199
+ homepage: parsed.data.homepage,
200
+ license: parsed.data.license,
201
+ raw: parsed.data,
202
+ };
203
+ }
@@ -1,18 +0,0 @@
1
- import { describe, expect, test } from "bun:test";
2
-
3
- import { setOverridesForTesting } from "../__tests__/feature-flag-test-helpers.js";
4
- import type { AssistantConfig } from "../config/schema.js";
5
- import { isAssistantDesktopEnabled } from "./desktop-feature.js";
6
-
7
- describe("isAssistantDesktopEnabled", () => {
8
- const config = {} as AssistantConfig;
9
-
10
- test("needs both the flag and a containerized runtime", () => {
11
- setOverridesForTesting({ "assistant-desktop": true });
12
- expect(isAssistantDesktopEnabled(config, true)).toBe(true);
13
- expect(isAssistantDesktopEnabled(config, false)).toBe(false);
14
-
15
- setOverridesForTesting({ "assistant-desktop": false });
16
- expect(isAssistantDesktopEnabled(config, true)).toBe(false);
17
- });
18
- });