@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/docs/desktop-browser-cli.md +33 -0
  2. package/openapi.yaml +17 -5
  3. package/package.json +1 -1
  4. package/scripts/smoke-desktop-browser-cli.ts +279 -0
  5. package/src/__tests__/config-schema.test.ts +1 -1
  6. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  7. package/src/__tests__/conversation-error.test.ts +4 -1
  8. package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
  9. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  10. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  11. package/src/__tests__/host-cu-proxy.test.ts +116 -18
  12. package/src/__tests__/list-all-apps.test.ts +40 -8
  13. package/src/__tests__/mcp-auth-routes.test.ts +99 -0
  14. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  15. package/src/__tests__/mcp-health-check.test.ts +32 -32
  16. package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
  17. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  18. package/src/__tests__/skills.test.ts +10 -0
  19. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  20. package/src/apps/app-store.ts +6 -5
  21. package/src/browser/operations.ts +2 -1
  22. package/src/browser/types.ts +7 -0
  23. package/src/browser/virtual-desktop-target.ts +34 -0
  24. package/src/calls/__tests__/progress-narration.test.ts +23 -0
  25. package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
  26. package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
  27. package/src/calls/progress-narration.ts +15 -2
  28. package/src/calls/voice-control-protocol.ts +34 -4
  29. package/src/calls/voice-session-bridge.ts +23 -0
  30. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  31. package/src/cli/commands/browser.help.ts +43 -0
  32. package/src/cli/commands/browser.ts +52 -8
  33. package/src/cli/commands/plugins.ts +16 -5
  34. package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
  35. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  36. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  37. package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
  38. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  39. package/src/cli/lib/bundled-marketplace.json +13 -0
  40. package/src/cli/lib/install-from-github.ts +17 -18
  41. package/src/cli/lib/install-from-platform.ts +14 -9
  42. package/src/cli/lib/list-installed-plugins.ts +21 -35
  43. package/src/cli/lib/plugin-details.ts +21 -8
  44. package/src/cli/lib/uninstall-plugin.ts +59 -1
  45. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  46. package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
  47. package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
  48. package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
  49. package/src/config/feature-flag-registry.json +2 -2
  50. package/src/config/schemas/__tests__/voice.test.ts +2 -2
  51. package/src/config/schemas/mcp.ts +9 -7
  52. package/src/config/schemas/monitoring.ts +9 -0
  53. package/src/config/schemas/voice.ts +2 -2
  54. package/src/config/skills.ts +13 -21
  55. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  56. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
  57. package/src/daemon/conversation-client-surface.ts +26 -0
  58. package/src/daemon/conversation-runtime-assembly.ts +1 -1
  59. package/src/daemon/conversation-tool-setup.ts +31 -29
  60. package/src/daemon/host-cu-proxy.ts +51 -26
  61. package/src/daemon/host-proxy-preactivation.ts +2 -0
  62. package/src/daemon/mcp-reload-service.ts +7 -0
  63. package/src/daemon/message-types/sync.ts +1 -0
  64. package/src/daemon/providers-setup.ts +4 -0
  65. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  66. package/src/desktop/desktop-automation-lease.test.ts +208 -0
  67. package/src/desktop/desktop-automation-lease.ts +270 -0
  68. package/src/desktop/desktop-browser-client.test.ts +413 -0
  69. package/src/desktop/desktop-browser-client.ts +430 -0
  70. package/src/desktop/desktop-browser-cursor.ts +40 -0
  71. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  72. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  73. package/src/desktop/desktop-browser-operations.ts +130 -0
  74. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  75. package/src/desktop/desktop-chrome-session.ts +58 -8
  76. package/src/desktop/desktop-dependencies.test.ts +29 -0
  77. package/src/desktop/desktop-dependencies.ts +26 -5
  78. package/src/desktop/desktop-display.ts +9 -0
  79. package/src/desktop/desktop-panel-config.ts +14 -1
  80. package/src/desktop/desktop-session-manager.test.ts +86 -3
  81. package/src/desktop/desktop-session-manager.ts +144 -24
  82. package/src/desktop/desktop-stream-bridge.ts +1 -1
  83. package/src/desktop/virtual-desktop-feature.ts +18 -0
  84. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  85. package/src/i18n/__tests__/i18n.test.ts +9 -5
  86. package/src/i18n/messages.ts +13 -4
  87. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  88. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  89. package/src/ipc/assistant-server.ts +15 -1
  90. package/src/ipc/cli-client.ts +13 -5
  91. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
  92. package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
  93. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  94. package/src/live-voice/__tests__/protocol.test.ts +11 -3
  95. package/src/live-voice/__tests__/session-controls.test.ts +34 -0
  96. package/src/live-voice/live-voice-metrics.ts +9 -0
  97. package/src/live-voice/live-voice-session.ts +3 -0
  98. package/src/live-voice/protocol.ts +10 -1
  99. package/src/live-voice/session-controls.ts +28 -0
  100. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  101. package/src/mcp/__tests__/effective-config.test.ts +11 -0
  102. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  103. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  104. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  105. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  106. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  107. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  108. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  109. package/src/mcp/client.ts +64 -16
  110. package/src/mcp/credential-target.ts +105 -0
  111. package/src/mcp/effective-config.ts +42 -15
  112. package/src/mcp/manager.ts +61 -8
  113. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  114. package/src/mcp/mcp-auth-state.ts +38 -6
  115. package/src/mcp/mcp-oauth-provider.ts +101 -33
  116. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  117. package/src/monitoring/mount-watch.ts +348 -0
  118. package/src/monitoring/worker.ts +9 -0
  119. package/src/oauth/seed-providers.ts +11 -0
  120. package/src/onboarding/checkin-event.ts +2 -1
  121. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  122. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  123. package/src/plugins/external-plugin-loader.ts +49 -86
  124. package/src/plugins/installed-plugin-dirs.ts +6 -4
  125. package/src/plugins/mcp-servers.ts +9 -15
  126. package/src/plugins/mtime-cache.ts +2 -1
  127. package/src/plugins/user-loader.ts +2 -1
  128. package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
  129. package/src/providers/inference/endpoint-probe.ts +27 -12
  130. package/src/providers/openai/responses-provider.ts +9 -2
  131. package/src/providers/opencode/client.test.ts +167 -0
  132. package/src/providers/opencode/client.ts +73 -4
  133. package/src/runtime/AGENTS.md +4 -0
  134. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
  135. package/src/runtime/http-server.ts +6 -6
  136. package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
  137. package/src/runtime/routes/browser-context.ts +39 -0
  138. package/src/runtime/routes/browser-routes.ts +16 -47
  139. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  140. package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
  141. package/src/runtime/routes/desktop-setup-routes.ts +7 -5
  142. package/src/runtime/routes/mcp-auth-routes.ts +106 -94
  143. package/src/runtime/routes/plugins-routes.ts +14 -1
  144. package/src/runtime/sync/resource-sync-events.ts +4 -0
  145. package/src/tools/browser/browser-execution.ts +44 -13
  146. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  147. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  148. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  149. package/src/tools/computer-use/definitions.ts +2 -3
  150. package/src/util/abort-reasons.ts +12 -1
  151. package/src/util/plugin-manifest.ts +203 -0
  152. package/src/desktop/desktop-feature.test.ts +0 -18
  153. package/src/desktop/desktop-feature.ts +0 -19
@@ -0,0 +1,33 @@
1
+ # Virtual desktop browser CLI
2
+
3
+ `assistant browser --virtual-desktop` uses the browser CLI's shared operation handlers against the Chrome process owned by `DesktopSessionManager`. It requires a platform-hosted container, the `assistant-desktop` flag, completed automatic desktop installation and an identified guardian conversation. Availability is checked before starting or reusing control and after asynchronous startup; loss of the flag or installation readiness cancels active control. The first eligible browser command waits for the shared installation job, starts Chrome, and executes the original action exactly once. Cancellation and feature revocation stop the pending action while shared installation can finish for other viewers. Browser CLI requests allow ten minutes, with an eight-minute setup wait limit. Eligible web conversations receive browser guidance in the bash tool description, including the configured shell timeout; native clients retain their existing tool description. Opening the picture-in-picture panel also starts setup and shows progress automatically; failed installs retain a retry action. The persistent profile remains `data/desktop-profile`; no data migration or extension installation is required.
4
+
5
+ Automatic browser selection in platform-hosted web conversations uses the streamed Chrome when the desktop flag is enabled and the actor is an identified guardian. The macOS and Windows apps keep their existing browser selection and fallback behavior; they use the streamed desktop only with explicit `--virtual-desktop`. Their shared renderer reports a `web` transport, so selection also reads the active turn's frozen `clientOs`; it does not change transport identity or host capability checks. Explicit `--virtual-desktop`, `--browser-mode`, `--target-client-id` and active-tab requests override the default. Existing personal-browser sessions and tab pins remain on their selected browser. Tab commands use the same streamed-browser default as page commands. `--browser-mode local` selects assistant-side Playwright, not the user's Chrome.
6
+
7
+ The legacy `--desktop` option remains an alias. The deployment flag key `assistant-desktop`, API paths, sync tags and persistent profile path are unchanged for compatibility. The shared `isVirtualDesktopEnabled` gate rejects self-hosted assistants, including local Docker, for setup, streaming and control.
8
+
9
+ ```mermaid
10
+ flowchart LR
11
+ CLI[assistant browser --virtual-desktop] --> IPC[Existing browser IPC routes]
12
+ IPC --> Lease[DesktopAutomationLease conversation and actor lease]
13
+ Lease --> Shared[Shared browser operation handlers]
14
+ Shared --> CDP[Scoped desktop CDP client]
15
+ CDP --> Chrome[Managed Chrome on display :99]
16
+ Chrome --> Stream[Existing desktop stream]
17
+ ```
18
+
19
+ Chrome and its dock launcher share the managed profile and loopback debug port. For Chrome reopened from the dock, the profile singleton lock identifies a candidate PID, validated against the installed executable, profile and loopback debug arguments. Discovery checks `/proc` socket ownership against that browser PID, refuses redirects and validates the returned browser WebSocket endpoint. The connection stays within the container. No CDP endpoint or token is exposed to the renderer.
20
+
21
+ The desktop client bypasses personal-browser discovery, extension reconnect waits and backend fallback. It reuses the existing AX snapshot, DOM element resolution, mouse, keyboard, extraction and credential-fill implementations. Operation-scoped clients borrow the lease's connection; disposing one does not release the lease. `detach`, `close`, cancellation, errors and idle expiry release control. Only `tabs close` closes a Chrome tab.
22
+
23
+ `assistant browser --virtual-desktop screenshot --output /tmp/desktop-page.jpg` captures a color page image through CDP `Page.captureScreenshot`. It excludes the browser toolbar and desktop dock. `snapshot` returns semantic page structure. Agents should use these interfaces and let the manager start the desktop, including when older memory notes describe manual Xvnc, `xdotool` or scratch XWD conversion scripts.
24
+
25
+ Tab IDs are ephemeral numeric aliases for this managed browser's CDP target IDs. They are never personal extension tab IDs. Initial attachment selects an existing HTTP(S) page or opens a blank tab. Use `tabs list` and `tabs select --tab-id <id>` to choose explicitly. Navigation, tab selection, document replacement and release clear the desktop snapshot map. Personal-browser snapshots use a separate namespace.
26
+
27
+ CDP mouse events use page viewport CSS coordinates. Before dispatching them, the client animates a purple arrow overlay to the same point. The overlay is excluded from accessibility and hit testing. It appears in the existing desktop stream and is removed on release. It does not move the OS pointer, appear in browser toolbar UI or visualize every programmatic DOM operation. Native dialogs and other applications are outside the browser CLI; users can interact with them directly in the expanded desktop. The picture-in-picture preview is view-only. Direct user interaction does not pause automation.
28
+
29
+ The client records key and mouse presses before dispatch. On release it opens a fresh bounded cleanup connection, attaches to the same live targets, releases uncertain held input and removes overlays. A failed cleanup preserves state and the automation slot for retry. Dispatched actions are never automatically retried. Closed targets need no input cleanup. Browser-process loss disposes the client, and later requests discover the replacement process.
30
+
31
+ `--use-active-tab` and personal browser targeting are rejected with `--virtual-desktop`. Download waiting is unsupported. Browser operations are bounded to two minutes and share the desktop lease's action budget and idle expiry.
32
+
33
+ Validation: focused client tests exercise shared snapshot/click behavior, namespace isolation, stale references, target changes, cancellation and uncertain-input cleanup. Lease tests cover browser ownership, cancellation and cleanup independently of native input. The Linux smoke script exercises real Chrome, the CLI and visible pointer feedback.
package/openapi.yaml CHANGED
@@ -3,7 +3,7 @@
3
3
  openapi: 3.1.0
4
4
  info:
5
5
  title: Vellum Assistant API
6
- version: 0.12.1
6
+ version: 0.12.2
7
7
  description: Auto-generated OpenAPI specification for the Vellum Assistant runtime HTTP server.
8
8
  servers:
9
9
  - url: http://127.0.0.1:7821
@@ -4146,6 +4146,8 @@ paths:
4146
4146
  conversationId:
4147
4147
  type: string
4148
4148
  minLength: 1
4149
+ desktop:
4150
+ type: boolean
4149
4151
  required:
4150
4152
  - operation
4151
4153
  responses:
@@ -4181,7 +4183,7 @@ paths:
4181
4183
  post:
4182
4184
  operationId: browser_tabs_post
4183
4185
  summary: Manage browser tabs
4184
- description: List, create, select, or close browser tabs via the Chrome extension backend.
4186
+ description: List, create, select, or close browser tabs in the Chrome extension or streamed desktop browser.
4185
4187
  tags:
4186
4188
  - browser
4187
4189
  requestBody:
@@ -4205,6 +4207,10 @@ paths:
4205
4207
  conversationId:
4206
4208
  type: string
4207
4209
  minLength: 1
4210
+ desktop:
4211
+ type: boolean
4212
+ browserMode:
4213
+ type: string
4208
4214
  tabId:
4209
4215
  type: number
4210
4216
  url:
@@ -11916,7 +11922,7 @@ paths:
11916
11922
  /v1/desktop/setup:
11917
11923
  get:
11918
11924
  operationId: desktop_setup_get
11919
- summary: Get desktop setup status
11925
+ summary: Get virtual desktop setup status
11920
11926
  tags:
11921
11927
  - desktop
11922
11928
  responses:
@@ -11946,7 +11952,7 @@ paths:
11946
11952
  additionalProperties: false
11947
11953
  post:
11948
11954
  operationId: desktop_setup_post
11949
- summary: Install desktop components
11955
+ summary: Install virtual desktop components
11950
11956
  tags:
11951
11957
  - desktop
11952
11958
  responses:
@@ -18426,7 +18432,7 @@ paths:
18426
18432
  get:
18427
18433
  operationId: internal_mcp_list_get
18428
18434
  summary: List MCP servers with health status
18429
- description: Returns configured MCP servers with live health-check results (connected, needs auth, error).
18435
+ description: Returns configured MCP servers with recorded runtime status without opening new connections.
18430
18436
  tags:
18431
18437
  - internal
18432
18438
  responses:
@@ -26544,6 +26550,12 @@ paths:
26544
26550
  target:
26545
26551
  type: string
26546
26552
  description: Absolute path that was removed on the assistant host. Useful for audit logs and confirmation toasts.
26553
+ warnings:
26554
+ description: Stable keys for non-fatal cleanup limitations that should be localized at the presentation edge.
26555
+ type: array
26556
+ items:
26557
+ type: string
26558
+ const: plugin.uninstall.mcp_oauth_credentials_unchecked
26547
26559
  required:
26548
26560
  - name
26549
26561
  - target
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vellumai/assistant",
3
- "version": "0.12.2-staging.2",
3
+ "version": "0.12.2-staging.3",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "exports": {
@@ -0,0 +1,279 @@
1
+ import assert from "node:assert/strict";
2
+ import { mkdir, mkdtemp, rm } from "node:fs/promises";
3
+ import { createServer } from "node:net";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+
7
+ import { IpcFrameReader, writeMessage } from "@vellumai/ipc-server-utils";
8
+ import { Command } from "commander";
9
+ import sharp from "sharp";
10
+
11
+ import { executeBrowserOperation } from "../src/browser/operations.js";
12
+ import type { BrowserOperation } from "../src/browser/types.js";
13
+ import { registerBrowserCommand } from "../src/cli/commands/browser.js";
14
+ import { DesktopAutomationLease } from "../src/desktop/desktop-automation-lease.js";
15
+ import {
16
+ desktopChromePath,
17
+ desktopDependencyInstaller,
18
+ } from "../src/desktop/desktop-dependencies.js";
19
+ import { DesktopSessionManager } from "../src/desktop/desktop-session-manager.js";
20
+ import { getAssistantSocketPath } from "../src/ipc/socket-path.js";
21
+
22
+ if (
23
+ process.platform !== "linux" ||
24
+ !process.argv[2] ||
25
+ !process.env.ASSISTANT_IPC_SOCKET_DIR
26
+ ) {
27
+ throw new Error(
28
+ "Run in disposable Linux with desktop packages, a temporary ASSISTANT_IPC_SOCKET_DIR and a Chrome executable argument or --install for a cold install.",
29
+ );
30
+ }
31
+ const coldInstall = process.argv[2] === "--install";
32
+ const executable = coldInstall ? desktopChromePath() : process.argv[2];
33
+ if (coldInstall) {
34
+ assert.equal(desktopDependencyInstaller.getStatus().state, "required");
35
+ assert.equal(Bun.which("Xtigervnc"), null);
36
+ }
37
+ const directory = await mkdtemp(join(tmpdir(), "desktop-browser-cli-"));
38
+ const manager = new DesktopSessionManager({
39
+ resolveChromePath: async () => executable,
40
+ profileDir: join(directory, "profile"),
41
+ panelConfigDir: join(directory, "panel"),
42
+ renderWallpaper: async () => null,
43
+ });
44
+ const control = new DesktopAutomationLease({
45
+ enabled: () => true,
46
+ ready: () =>
47
+ !coldInstall || desktopDependencyInstaller.getStatus().state === "ready",
48
+ ensureReady: (signal) => desktopDependencyInstaller.ensureReady(signal),
49
+ manager: () => manager,
50
+ });
51
+ const context = {
52
+ workingDir: directory,
53
+ conversationId: "conv-smoke",
54
+ sourceActorPrincipalId: "user-123",
55
+ trustClass: "guardian" as const,
56
+ };
57
+ let submissions = 0;
58
+ let navigations = 0;
59
+ const page = Bun.serve({
60
+ hostname: "127.0.0.1",
61
+ port: 0,
62
+ fetch(request) {
63
+ if (new URL(request.url).pathname === "/save") {
64
+ submissions++;
65
+ return Response.json({ ok: true });
66
+ }
67
+ if (new URL(request.url).pathname === "/") {
68
+ navigations++;
69
+ }
70
+ return new Response(
71
+ `<!doctype html><html><head><title>Desktop browser CLI</title><style>body{font:24px sans-serif;padding:70px;background:#f3f4f8}input,button{font:inherit;padding:12px;margin:16px}#result{color:#5140bd}#colors{position:fixed;left:0;top:0;display:flex}#colors span{width:100px;height:50px}</style></head><body><div id="colors"><span style="background:#ff0000"></span><span style="background:#00ff00"></span><span style="background:#0000ff"></span></div><h1>Streamed desktop browser</h1><label>Example text<input id="text"></label><button id="save" onclick="fetch('/save');document.getElementById('result').textContent='Saved '+document.getElementById('text').value">Save</button><p id="result"></p><button id="open-dialog" onclick="document.querySelector('dialog').showModal()">Open dialog</button><dialog style="background:white"><button id="inside" onclick="this.textContent='Modal clicked'">Inside dialog</button></dialog></body></html>`,
72
+ {
73
+ headers: {
74
+ "content-type": "text/html",
75
+ "content-security-policy":
76
+ "require-trusted-types-for 'script'; trusted-types 'none'",
77
+ },
78
+ },
79
+ );
80
+ },
81
+ });
82
+ await mkdir(process.env.ASSISTANT_IPC_SOCKET_DIR, { recursive: true });
83
+ let setupNotifications = 0;
84
+ const ipc = createServer((socket) => {
85
+ const reader = new IpcFrameReader((request) => {
86
+ void (async () => {
87
+ try {
88
+ if (request.method === "/events/publish") {
89
+ setupNotifications++;
90
+ writeMessage(socket, { id: request.id, result: { ok: true } });
91
+ return;
92
+ }
93
+ const body = (
94
+ request.params as {
95
+ body: {
96
+ operation: BrowserOperation;
97
+ input: Record<string, unknown>;
98
+ desktop: boolean;
99
+ };
100
+ }
101
+ ).body;
102
+ assert.equal(request.method, "browser_execute");
103
+ assert.equal(body.desktop, true);
104
+ const result = await control.runBrowser(
105
+ context,
106
+ async (signal) => {
107
+ const cdpClient = await manager.browser.client(
108
+ context.conversationId,
109
+ signal,
110
+ );
111
+ return executeBrowserOperation(body.operation, body.input, {
112
+ ...context,
113
+ signal,
114
+ cdpClient,
115
+ });
116
+ },
117
+ body.operation === "detach",
118
+ );
119
+ const screenshots = (result.contentBlocks ?? []).flatMap((block) =>
120
+ block.type === "image" && block.source.type === "base64"
121
+ ? [{ mediaType: block.source.media_type, data: block.source.data }]
122
+ : [],
123
+ );
124
+ writeMessage(socket, {
125
+ id: request.id,
126
+ result: { ...result, screenshots },
127
+ });
128
+ } catch (error) {
129
+ writeMessage(socket, { id: request.id, error: String(error) });
130
+ }
131
+ })();
132
+ });
133
+ socket.on("data", (chunk) => reader.push(chunk));
134
+ });
135
+ await new Promise<void>((resolve, reject) => {
136
+ ipc.once("error", reject);
137
+ ipc.listen(getAssistantSocketPath(), resolve);
138
+ });
139
+ async function cli(...args: string[]) {
140
+ const program = new Command();
141
+ program.exitOverride();
142
+ registerBrowserCommand(program);
143
+ const chunks: string[] = [];
144
+ const original = process.stdout.write;
145
+ process.stdout.write = ((chunk: unknown) => {
146
+ chunks.push(String(chunk));
147
+ return true;
148
+ }) as typeof original;
149
+ try {
150
+ await program.parseAsync([
151
+ "bun",
152
+ "assistant",
153
+ "browser",
154
+ "--virtual-desktop",
155
+ "--json",
156
+ ...args,
157
+ ]);
158
+ } finally {
159
+ process.stdout.write = original;
160
+ }
161
+ const result = JSON.parse(chunks.join("")) as {
162
+ ok: boolean;
163
+ content: string;
164
+ error?: string;
165
+ };
166
+ assert.equal(result.ok, true, JSON.stringify(result));
167
+ return result.content;
168
+ }
169
+ function ref(snapshot: string, name: string): string {
170
+ const line = snapshot
171
+ .split("\n")
172
+ .find((value) => value.includes(name) && /\be\d+\b/.test(value));
173
+ assert(line, `Missing ${name}: ${snapshot}`);
174
+ return /\be\d+\b/.exec(line)![0];
175
+ }
176
+ try {
177
+ await cli(
178
+ "navigate",
179
+ "--url",
180
+ `http://127.0.0.1:${page.port}`,
181
+ "--allow-private-network",
182
+ );
183
+ assert.equal(navigations, 1);
184
+ if (coldInstall) {
185
+ assert.equal(desktopDependencyInstaller.getStatus().state, "ready");
186
+ assert(
187
+ setupNotifications >= 3,
188
+ "Installation progress must reach the viewer",
189
+ );
190
+ console.error(
191
+ "PASS: one CLI navigate installed desktop and Chrome from scratch, then loaded the requested page once",
192
+ );
193
+ }
194
+ const snapshot = await cli("snapshot");
195
+ await cli(
196
+ "type",
197
+ "--element-id",
198
+ ref(snapshot, "Example text"),
199
+ "--text",
200
+ "Hello café 世界",
201
+ "--clear-first",
202
+ );
203
+ await cli("click", "--element-id", ref(snapshot, "Save"));
204
+ const saved = await cli("extract");
205
+ assert.match(saved, /Saved Hello café 世界/);
206
+ assert.equal(submissions, 1);
207
+ const screenshotPath = join(tmpdir(), "desktop-browser-cli-page.jpg");
208
+ await cli("screenshot", "--output", screenshotPath);
209
+ const { data: pixels, info } = await sharp(screenshotPath)
210
+ .removeAlpha()
211
+ .raw()
212
+ .toBuffer({ resolveWithObject: true });
213
+ for (let channel = 0; channel < 3; channel++) {
214
+ const offset = (25 * info.width + 50 + channel * 100) * info.channels;
215
+ for (let component = 0; component < 3; component++) {
216
+ const expected = component === channel ? 255 : 0;
217
+ assert(
218
+ Math.abs(pixels[offset + component] - expected) < 10,
219
+ `Incorrect screenshot color for swatch ${channel}, channel ${component}`,
220
+ );
221
+ }
222
+ }
223
+ async function assertCursorPainted(expected: boolean) {
224
+ await control.runBrowser(context, async (signal) => {
225
+ const cdp = await manager.browser.client(context.conversationId, signal);
226
+ const screenshot = await cdp.send<{ data: string }>(
227
+ "Page.captureScreenshot",
228
+ { format: "png" },
229
+ );
230
+ const { data, info } = await sharp(Buffer.from(screenshot.data, "base64"))
231
+ .removeAlpha()
232
+ .raw()
233
+ .toBuffer({ resolveWithObject: true });
234
+ let purple = 0;
235
+ for (let i = 0; i < data.length; i += info.channels) {
236
+ if (
237
+ Math.abs(data[i]! - 112) < 5 &&
238
+ Math.abs(data[i + 1]! - 87) < 5 &&
239
+ Math.abs(data[i + 2]! - 255) < 5
240
+ ) {
241
+ purple++;
242
+ }
243
+ }
244
+ assert.equal(purple > 40, expected, `Cursor pixels: ${purple}`);
245
+ return { content: "verified", isError: false };
246
+ });
247
+ }
248
+ await assertCursorPainted(true);
249
+ await cli("click", "--selector", "#open-dialog");
250
+ await assertCursorPainted(true);
251
+ await cli("hover", "--selector", "#inside");
252
+ await assertCursorPainted(true);
253
+ await cli("click", "--selector", "#inside");
254
+ assert.match(await cli("extract"), /Modal clicked/);
255
+ for (const key of ["Enter", "Space"]) {
256
+ await cli("press-key", "--key", "Escape");
257
+ await cli("press-key", "--key", key, "--selector", "#open-dialog");
258
+ await assertCursorPainted(true);
259
+ }
260
+ await cli(
261
+ "navigate",
262
+ "--url",
263
+ `http://127.0.0.1:${page.port}/next`,
264
+ "--allow-private-network",
265
+ );
266
+ await assertCursorPainted(true);
267
+ await cli("detach");
268
+ await assertCursorPainted(false);
269
+ await cli("detach");
270
+ console.log(
271
+ "PASS: real browser CLI over IPC, shared AX snapshot, Unicode typing, one click submission, RGB page screenshot, cursor pixels above dialogs and after navigation, detach cleanup",
272
+ );
273
+ } finally {
274
+ await cli("detach");
275
+ await manager.destroy();
276
+ await new Promise<void>((resolve) => ipc.close(() => resolve()));
277
+ page.stop(true);
278
+ await rm(directory, { recursive: true, force: true });
279
+ }
@@ -1021,7 +1021,7 @@ describe("AssistantConfigSchema", () => {
1021
1021
  maxSilenceMs: 35000,
1022
1022
  longOpMs: 15000,
1023
1023
  minGapMs: 6000,
1024
- generationTimeoutMs: 1500,
1024
+ generationTimeoutMs: 5000,
1025
1025
  },
1026
1026
  },
1027
1027
  });
@@ -5,6 +5,7 @@ import { SYNC_TAGS } from "../daemon/message-types/sync.js";
5
5
  import { assistantEventHub } from "../runtime/assistant-event-hub.js";
6
6
  import {
7
7
  publishConfigChanged,
8
+ publishMcpChanged,
8
9
  publishSchedulesChanged,
9
10
  publishSoundsConfigUpdated,
10
11
  } from "../runtime/sync/resource-sync-events.js";
@@ -94,4 +95,26 @@ describe("config and sounds sync events", () => {
94
95
  subscription.dispose();
95
96
  }
96
97
  });
98
+
99
+ test("MCP changes emit a sync tag without a client origin", async () => {
100
+ const received: AssistantEventEnvelope[] = [];
101
+ const subscription = assistantEventHub.subscribe({
102
+ type: "process",
103
+ callback: (event) => {
104
+ received.push(event);
105
+ },
106
+ });
107
+
108
+ try {
109
+ publishMcpChanged();
110
+ await waitFor(() => received.length === 1);
111
+
112
+ expect(received[0].message).toEqual({
113
+ type: "sync_changed",
114
+ tags: [SYNC_TAGS.mcpList],
115
+ });
116
+ } finally {
117
+ subscription.dispose();
118
+ }
119
+ });
97
120
  });
@@ -1490,6 +1490,7 @@ describe("classifyConversationError", () => {
1490
1490
  "subagent_aborted",
1491
1491
  "signal_cancel",
1492
1492
  "voice_session_aborted",
1493
+ "voice_progress_narration_timeout",
1493
1494
  ];
1494
1495
 
1495
1496
  for (const kind of taggedKinds) {
@@ -1727,7 +1728,9 @@ describe("ConnectionResolutionError classification", () => {
1727
1728
  expect(result.userMessage).toContain("qwen/qwen3-8b");
1728
1729
  expect(result.userMessage).toContain("Vellum GPU route");
1729
1730
  expect(result.userMessage).toContain('profile "steer"');
1730
- expect(result.userMessage).toContain("was not sent through another provider");
1731
+ expect(result.userMessage).toContain(
1732
+ "was not sent through another provider",
1733
+ );
1731
1734
  });
1732
1735
 
1733
1736
  it("classifies missing_credential naming the connection and fix", () => {
@@ -607,7 +607,9 @@ describe("injectChannelCapabilityContext", () => {
607
607
  "ui_show, ui_update, and ui_dismiss persist conversation content",
608
608
  );
609
609
  expect(text).not.toContain("Only use ui_show/ui_update");
610
- expect(text).not.toContain("Do NOT use ui_show, ui_update, or app_create");
610
+ expect(text).not.toContain(
611
+ "Do NOT use ui_show, ui_update, or app_create",
612
+ );
611
613
  expect(text).not.toContain(
612
614
  "Present information as well-formatted text instead of dynamic UI.",
613
615
  );
@@ -898,12 +900,12 @@ describe("trust-gating via channel capabilities", () => {
898
900
 
899
901
  const result = injectChannelCapabilityContext(message, caps);
900
902
 
901
- // macOS clients now get osascript guidance injected
903
+ // macOS clients get app-scripting guidance injected
902
904
  expect(result).not.toBe(message);
903
905
  const injected = (result.content[0] as { type: "text"; text: string }).text;
904
906
  expect(injected).toContain("client_os: macos");
905
- expect(injected).toContain("osascript");
906
- expect(injected).toContain("host_bash");
907
+ expect(injected).toContain("drive apps with the computer-use skill");
908
+ expect(injected).toContain("`host_bash` is for shell commands");
907
909
  // No channel constraints — full desktop capabilities
908
910
  expect(injected).not.toContain("CHANNEL CONSTRAINTS");
909
911
  });
@@ -39,6 +39,20 @@ function writePackageJson(dir: string, pkg: Record<string, unknown>): void {
39
39
  writeFileSync(join(dir, "package.json"), JSON.stringify(pkg, null, 2));
40
40
  }
41
41
 
42
+ function writePluginJson(dir: string, manifest: Record<string, unknown>): void {
43
+ writeFileSync(
44
+ join(dir, "plugin.json"),
45
+ JSON.stringify(
46
+ {
47
+ $schema: "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
48
+ ...manifest,
49
+ },
50
+ null,
51
+ 2,
52
+ ),
53
+ );
54
+ }
55
+
42
56
  function writeSurfaceFile(dir: string, relPath: string, body: string): void {
43
57
  const parts = relPath.split("/");
44
58
  parts.pop();
@@ -104,6 +118,31 @@ describe("loadExternalPlugin — manifest", () => {
104
118
  expect(registered).toBeDefined();
105
119
  expect(registered?.manifest.version).toBe("0.0.0");
106
120
  });
121
+
122
+ test("loads a standard plugin.json when package.json is absent", async () => {
123
+ const dir = freshPluginDir("standard-only");
124
+ writePluginJson(dir, { name: "standard-only", version: "2.1.0" });
125
+
126
+ await loadExternalPlugin(dir);
127
+
128
+ const registered = getRegisteredPlugins().find(
129
+ (plugin) => plugin.manifest.name === "standard-only",
130
+ );
131
+ expect(registered?.manifest.version).toBe("2.1.0");
132
+ });
133
+
134
+ test("keeps package.json authoritative when plugin.json is also present", async () => {
135
+ const dir = freshPluginDir("mixed-manifests");
136
+ writePackageJson(dir, { name: "legacy-name", version: "1.2.3" });
137
+ writeFileSync(join(dir, "plugin.json"), "{ foreign and malformed }");
138
+
139
+ await loadExternalPlugin(dir);
140
+
141
+ const registered = getRegisteredPlugins().find(
142
+ (plugin) => plugin.manifest.name === "mixed-manifests",
143
+ );
144
+ expect(registered?.manifest.version).toBe("1.2.3");
145
+ });
107
146
  });
108
147
 
109
148
  describe("credentialKeyPatterns manifest field", () => {
@@ -277,6 +316,19 @@ describe("loadExternalPlugin — plugin-api peerDependency", () => {
277
316
  test("malformed package.json is logged and skipped (registry untouched)", async () => {
278
317
  const dir = freshPluginDir("malformed-pkg");
279
318
  writeFileSync(join(dir, "package.json"), "{ this is not json");
319
+ writePluginJson(dir, { name: "valid-standard-fallback" });
320
+
321
+ await loadExternalPlugin(dir);
322
+
323
+ expect(registeredNames()).toHaveLength(0);
324
+ });
325
+
326
+ test("malformed selected plugin.json is logged and skipped", async () => {
327
+ const dir = freshPluginDir("malformed-standard");
328
+ writeFileSync(
329
+ join(dir, "plugin.json"),
330
+ JSON.stringify({ name: "missing-schema" }),
331
+ );
280
332
 
281
333
  await loadExternalPlugin(dir);
282
334
 
@@ -41,6 +41,16 @@ function makeFakeCdp(
41
41
  dispose() {
42
42
  _cdpDisposed = true;
43
43
  },
44
+ setCdpSessionId() {},
45
+ async listTabs() {
46
+ return [];
47
+ },
48
+ async selectTab(tabId: number) {
49
+ return { tabId };
50
+ },
51
+ async closeTab(tabId: number) {
52
+ return { tabId, closed: true };
53
+ },
44
54
  };
45
55
  }
46
56
 
@@ -142,6 +152,8 @@ mock.module("../tools/network/url-safety.js", () => ({
142
152
 
143
153
  import {
144
154
  executeBrowserClick,
155
+ executeBrowserClose,
156
+ executeBrowserDetach,
145
157
  executeBrowserNavigate,
146
158
  executeBrowserScreenshot,
147
159
  executeBrowserSnapshot,
@@ -687,6 +699,42 @@ describe("browser_mode wiring through tool execution", () => {
687
699
 
688
700
  // ── Per-conversation sticky backend kind ─────────────────────────
689
701
 
702
+ for (const release of [executeBrowserDetach, executeBrowserClose]) {
703
+ test.each(["local", "extension", "cdp-inspect"])(
704
+ `${release.name} on desktop preserves the normal %s session's backend`,
705
+ async (mode) => {
706
+ const first = await executeBrowserSnapshot({ browser_mode: mode }, ctx);
707
+ expect(first.isError).toBe(false);
708
+
709
+ factoryModeCalls.length = 0;
710
+ const desktop = await release(
711
+ {},
712
+ {
713
+ ...ctx,
714
+ cdpClient: makeFakeCdp(
715
+ "cdp-inspect",
716
+ `desktop-browser:${ctx.conversationId}`,
717
+ ),
718
+ },
719
+ );
720
+ expect(desktop.isError).toBe(false);
721
+ expect(factoryModeCalls).toEqual([]);
722
+
723
+ const resumed = await executeBrowserSnapshot({}, ctx);
724
+ expect(resumed.isError).toBe(false);
725
+ expect(factoryModeCalls).toEqual([mode]);
726
+ },
727
+ );
728
+
729
+ test(`${release.name} on the normal browser still resets automatic selection`, async () => {
730
+ await executeBrowserSnapshot({ browser_mode: "extension" }, ctx);
731
+ expect((await release({}, ctx)).isError).toBe(false);
732
+ factoryModeCalls.length = 0;
733
+ expect((await executeBrowserSnapshot({}, ctx)).isError).toBe(false);
734
+ expect(factoryModeCalls).toEqual(["auto"]);
735
+ });
736
+ }
737
+
690
738
  test("auto-mode call after an explicit pin sticks to the pinned kind", async () => {
691
739
  const first = await executeBrowserNavigate(
692
740
  { url: "https://example.com", browser_mode: "local" },