@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/desktop-browser-cli.md +33 -0
- package/openapi.yaml +17 -5
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +279 -0
- package/src/__tests__/config-schema.test.ts +1 -1
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/host-cu-proxy.test.ts +116 -18
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/mcp-auth-routes.test.ts +99 -0
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-health-check.test.ts +32 -32
- package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/skills.test.ts +10 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/apps/app-store.ts +6 -5
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/calls/__tests__/progress-narration.test.ts +23 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
- package/src/calls/progress-narration.ts +15 -2
- package/src/calls/voice-control-protocol.ts +34 -4
- package/src/calls/voice-session-bridge.ts +23 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/plugins.ts +16 -5
- package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +13 -0
- package/src/cli/lib/install-from-github.ts +17 -18
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-details.ts +21 -8
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
- package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
- package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
- package/src/config/feature-flag-registry.json +2 -2
- package/src/config/schemas/__tests__/voice.test.ts +2 -2
- package/src/config/schemas/mcp.ts +9 -7
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/voice.ts +2 -2
- package/src/config/skills.ts +13 -21
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-runtime-assembly.ts +1 -1
- package/src/daemon/conversation-tool-setup.ts +31 -29
- package/src/daemon/host-cu-proxy.ts +51 -26
- package/src/daemon/host-proxy-preactivation.ts +2 -0
- package/src/daemon/mcp-reload-service.ts +7 -0
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/providers-setup.ts +4 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-automation-lease.test.ts +208 -0
- package/src/desktop/desktop-automation-lease.ts +270 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +58 -8
- package/src/desktop/desktop-dependencies.test.ts +29 -0
- package/src/desktop/desktop-dependencies.ts +26 -5
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.ts +14 -1
- package/src/desktop/desktop-session-manager.test.ts +86 -3
- package/src/desktop/desktop-session-manager.ts +144 -24
- package/src/desktop/desktop-stream-bridge.ts +1 -1
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/i18n/__tests__/i18n.test.ts +9 -5
- package/src/i18n/messages.ts +13 -4
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/protocol.test.ts +11 -3
- package/src/live-voice/__tests__/session-controls.test.ts +34 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-session.ts +3 -0
- package/src/live-voice/protocol.ts +10 -1
- package/src/live-voice/session-controls.ts +28 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +11 -0
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +42 -15
- package/src/mcp/manager.ts +61 -8
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/oauth/seed-providers.ts +11 -0
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +9 -15
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
- package/src/providers/inference/endpoint-probe.ts +27 -12
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +167 -0
- package/src/providers/opencode/client.ts +73 -4
- package/src/runtime/AGENTS.md +4 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
- package/src/runtime/http-server.ts +6 -6
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
- package/src/runtime/routes/desktop-setup-routes.ts +7 -5
- package/src/runtime/routes/mcp-auth-routes.ts +106 -94
- package/src/runtime/routes/plugins-routes.ts +14 -1
- package/src/runtime/sync/resource-sync-events.ts +4 -0
- package/src/tools/browser/browser-execution.ts +44 -13
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/computer-use/definitions.ts +2 -3
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/plugin-manifest.ts +203 -0
- package/src/desktop/desktop-feature.test.ts +0 -18
- package/src/desktop/desktop-feature.ts +0 -19
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Virtual desktop browser CLI
|
|
2
|
+
|
|
3
|
+
`assistant browser --virtual-desktop` uses the browser CLI's shared operation handlers against the Chrome process owned by `DesktopSessionManager`. It requires a platform-hosted container, the `assistant-desktop` flag, completed automatic desktop installation and an identified guardian conversation. Availability is checked before starting or reusing control and after asynchronous startup; loss of the flag or installation readiness cancels active control. The first eligible browser command waits for the shared installation job, starts Chrome, and executes the original action exactly once. Cancellation and feature revocation stop the pending action while shared installation can finish for other viewers. Browser CLI requests allow ten minutes, with an eight-minute setup wait limit. Eligible web conversations receive browser guidance in the bash tool description, including the configured shell timeout; native clients retain their existing tool description. Opening the picture-in-picture panel also starts setup and shows progress automatically; failed installs retain a retry action. The persistent profile remains `data/desktop-profile`; no data migration or extension installation is required.
|
|
4
|
+
|
|
5
|
+
Automatic browser selection in platform-hosted web conversations uses the streamed Chrome when the desktop flag is enabled and the actor is an identified guardian. The macOS and Windows apps keep their existing browser selection and fallback behavior; they use the streamed desktop only with explicit `--virtual-desktop`. Their shared renderer reports a `web` transport, so selection also reads the active turn's frozen `clientOs`; it does not change transport identity or host capability checks. Explicit `--virtual-desktop`, `--browser-mode`, `--target-client-id` and active-tab requests override the default. Existing personal-browser sessions and tab pins remain on their selected browser. Tab commands use the same streamed-browser default as page commands. `--browser-mode local` selects assistant-side Playwright, not the user's Chrome.
|
|
6
|
+
|
|
7
|
+
The legacy `--desktop` option remains an alias. The deployment flag key `assistant-desktop`, API paths, sync tags and persistent profile path are unchanged for compatibility. The shared `isVirtualDesktopEnabled` gate rejects self-hosted assistants, including local Docker, for setup, streaming and control.
|
|
8
|
+
|
|
9
|
+
```mermaid
|
|
10
|
+
flowchart LR
|
|
11
|
+
CLI[assistant browser --virtual-desktop] --> IPC[Existing browser IPC routes]
|
|
12
|
+
IPC --> Lease[DesktopAutomationLease conversation and actor lease]
|
|
13
|
+
Lease --> Shared[Shared browser operation handlers]
|
|
14
|
+
Shared --> CDP[Scoped desktop CDP client]
|
|
15
|
+
CDP --> Chrome[Managed Chrome on display :99]
|
|
16
|
+
Chrome --> Stream[Existing desktop stream]
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Chrome and its dock launcher share the managed profile and loopback debug port. For Chrome reopened from the dock, the profile singleton lock identifies a candidate PID, validated against the installed executable, profile and loopback debug arguments. Discovery checks `/proc` socket ownership against that browser PID, refuses redirects and validates the returned browser WebSocket endpoint. The connection stays within the container. No CDP endpoint or token is exposed to the renderer.
|
|
20
|
+
|
|
21
|
+
The desktop client bypasses personal-browser discovery, extension reconnect waits and backend fallback. It reuses the existing AX snapshot, DOM element resolution, mouse, keyboard, extraction and credential-fill implementations. Operation-scoped clients borrow the lease's connection; disposing one does not release the lease. `detach`, `close`, cancellation, errors and idle expiry release control. Only `tabs close` closes a Chrome tab.
|
|
22
|
+
|
|
23
|
+
`assistant browser --virtual-desktop screenshot --output /tmp/desktop-page.jpg` captures a color page image through CDP `Page.captureScreenshot`. It excludes the browser toolbar and desktop dock. `snapshot` returns semantic page structure. Agents should use these interfaces and let the manager start the desktop, including when older memory notes describe manual Xvnc, `xdotool` or scratch XWD conversion scripts.
|
|
24
|
+
|
|
25
|
+
Tab IDs are ephemeral numeric aliases for this managed browser's CDP target IDs. They are never personal extension tab IDs. Initial attachment selects an existing HTTP(S) page or opens a blank tab. Use `tabs list` and `tabs select --tab-id <id>` to choose explicitly. Navigation, tab selection, document replacement and release clear the desktop snapshot map. Personal-browser snapshots use a separate namespace.
|
|
26
|
+
|
|
27
|
+
CDP mouse events use page viewport CSS coordinates. Before dispatching them, the client animates a purple arrow overlay to the same point. The overlay is excluded from accessibility and hit testing. It appears in the existing desktop stream and is removed on release. It does not move the OS pointer, appear in browser toolbar UI or visualize every programmatic DOM operation. Native dialogs and other applications are outside the browser CLI; users can interact with them directly in the expanded desktop. The picture-in-picture preview is view-only. Direct user interaction does not pause automation.
|
|
28
|
+
|
|
29
|
+
The client records key and mouse presses before dispatch. On release it opens a fresh bounded cleanup connection, attaches to the same live targets, releases uncertain held input and removes overlays. A failed cleanup preserves state and the automation slot for retry. Dispatched actions are never automatically retried. Closed targets need no input cleanup. Browser-process loss disposes the client, and later requests discover the replacement process.
|
|
30
|
+
|
|
31
|
+
`--use-active-tab` and personal browser targeting are rejected with `--virtual-desktop`. Download waiting is unsupported. Browser operations are bounded to two minutes and share the desktop lease's action budget and idle expiry.
|
|
32
|
+
|
|
33
|
+
Validation: focused client tests exercise shared snapshot/click behavior, namespace isolation, stale references, target changes, cancellation and uncertain-input cleanup. Lease tests cover browser ownership, cancellation and cleanup independently of native input. The Linux smoke script exercises real Chrome, the CLI and visible pointer feedback.
|
package/openapi.yaml
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
openapi: 3.1.0
|
|
4
4
|
info:
|
|
5
5
|
title: Vellum Assistant API
|
|
6
|
-
version: 0.12.
|
|
6
|
+
version: 0.12.2
|
|
7
7
|
description: Auto-generated OpenAPI specification for the Vellum Assistant runtime HTTP server.
|
|
8
8
|
servers:
|
|
9
9
|
- url: http://127.0.0.1:7821
|
|
@@ -4146,6 +4146,8 @@ paths:
|
|
|
4146
4146
|
conversationId:
|
|
4147
4147
|
type: string
|
|
4148
4148
|
minLength: 1
|
|
4149
|
+
desktop:
|
|
4150
|
+
type: boolean
|
|
4149
4151
|
required:
|
|
4150
4152
|
- operation
|
|
4151
4153
|
responses:
|
|
@@ -4181,7 +4183,7 @@ paths:
|
|
|
4181
4183
|
post:
|
|
4182
4184
|
operationId: browser_tabs_post
|
|
4183
4185
|
summary: Manage browser tabs
|
|
4184
|
-
description: List, create, select, or close browser tabs
|
|
4186
|
+
description: List, create, select, or close browser tabs in the Chrome extension or streamed desktop browser.
|
|
4185
4187
|
tags:
|
|
4186
4188
|
- browser
|
|
4187
4189
|
requestBody:
|
|
@@ -4205,6 +4207,10 @@ paths:
|
|
|
4205
4207
|
conversationId:
|
|
4206
4208
|
type: string
|
|
4207
4209
|
minLength: 1
|
|
4210
|
+
desktop:
|
|
4211
|
+
type: boolean
|
|
4212
|
+
browserMode:
|
|
4213
|
+
type: string
|
|
4208
4214
|
tabId:
|
|
4209
4215
|
type: number
|
|
4210
4216
|
url:
|
|
@@ -11916,7 +11922,7 @@ paths:
|
|
|
11916
11922
|
/v1/desktop/setup:
|
|
11917
11923
|
get:
|
|
11918
11924
|
operationId: desktop_setup_get
|
|
11919
|
-
summary: Get desktop setup status
|
|
11925
|
+
summary: Get virtual desktop setup status
|
|
11920
11926
|
tags:
|
|
11921
11927
|
- desktop
|
|
11922
11928
|
responses:
|
|
@@ -11946,7 +11952,7 @@ paths:
|
|
|
11946
11952
|
additionalProperties: false
|
|
11947
11953
|
post:
|
|
11948
11954
|
operationId: desktop_setup_post
|
|
11949
|
-
summary: Install desktop components
|
|
11955
|
+
summary: Install virtual desktop components
|
|
11950
11956
|
tags:
|
|
11951
11957
|
- desktop
|
|
11952
11958
|
responses:
|
|
@@ -18426,7 +18432,7 @@ paths:
|
|
|
18426
18432
|
get:
|
|
18427
18433
|
operationId: internal_mcp_list_get
|
|
18428
18434
|
summary: List MCP servers with health status
|
|
18429
|
-
description: Returns configured MCP servers with
|
|
18435
|
+
description: Returns configured MCP servers with recorded runtime status without opening new connections.
|
|
18430
18436
|
tags:
|
|
18431
18437
|
- internal
|
|
18432
18438
|
responses:
|
|
@@ -26544,6 +26550,12 @@ paths:
|
|
|
26544
26550
|
target:
|
|
26545
26551
|
type: string
|
|
26546
26552
|
description: Absolute path that was removed on the assistant host. Useful for audit logs and confirmation toasts.
|
|
26553
|
+
warnings:
|
|
26554
|
+
description: Stable keys for non-fatal cleanup limitations that should be localized at the presentation edge.
|
|
26555
|
+
type: array
|
|
26556
|
+
items:
|
|
26557
|
+
type: string
|
|
26558
|
+
const: plugin.uninstall.mcp_oauth_credentials_unchecked
|
|
26547
26559
|
required:
|
|
26548
26560
|
- name
|
|
26549
26561
|
- target
|
package/package.json
CHANGED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
|
3
|
+
import { createServer } from "node:net";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
import { IpcFrameReader, writeMessage } from "@vellumai/ipc-server-utils";
|
|
8
|
+
import { Command } from "commander";
|
|
9
|
+
import sharp from "sharp";
|
|
10
|
+
|
|
11
|
+
import { executeBrowserOperation } from "../src/browser/operations.js";
|
|
12
|
+
import type { BrowserOperation } from "../src/browser/types.js";
|
|
13
|
+
import { registerBrowserCommand } from "../src/cli/commands/browser.js";
|
|
14
|
+
import { DesktopAutomationLease } from "../src/desktop/desktop-automation-lease.js";
|
|
15
|
+
import {
|
|
16
|
+
desktopChromePath,
|
|
17
|
+
desktopDependencyInstaller,
|
|
18
|
+
} from "../src/desktop/desktop-dependencies.js";
|
|
19
|
+
import { DesktopSessionManager } from "../src/desktop/desktop-session-manager.js";
|
|
20
|
+
import { getAssistantSocketPath } from "../src/ipc/socket-path.js";
|
|
21
|
+
|
|
22
|
+
if (
|
|
23
|
+
process.platform !== "linux" ||
|
|
24
|
+
!process.argv[2] ||
|
|
25
|
+
!process.env.ASSISTANT_IPC_SOCKET_DIR
|
|
26
|
+
) {
|
|
27
|
+
throw new Error(
|
|
28
|
+
"Run in disposable Linux with desktop packages, a temporary ASSISTANT_IPC_SOCKET_DIR and a Chrome executable argument or --install for a cold install.",
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
const coldInstall = process.argv[2] === "--install";
|
|
32
|
+
const executable = coldInstall ? desktopChromePath() : process.argv[2];
|
|
33
|
+
if (coldInstall) {
|
|
34
|
+
assert.equal(desktopDependencyInstaller.getStatus().state, "required");
|
|
35
|
+
assert.equal(Bun.which("Xtigervnc"), null);
|
|
36
|
+
}
|
|
37
|
+
const directory = await mkdtemp(join(tmpdir(), "desktop-browser-cli-"));
|
|
38
|
+
const manager = new DesktopSessionManager({
|
|
39
|
+
resolveChromePath: async () => executable,
|
|
40
|
+
profileDir: join(directory, "profile"),
|
|
41
|
+
panelConfigDir: join(directory, "panel"),
|
|
42
|
+
renderWallpaper: async () => null,
|
|
43
|
+
});
|
|
44
|
+
const control = new DesktopAutomationLease({
|
|
45
|
+
enabled: () => true,
|
|
46
|
+
ready: () =>
|
|
47
|
+
!coldInstall || desktopDependencyInstaller.getStatus().state === "ready",
|
|
48
|
+
ensureReady: (signal) => desktopDependencyInstaller.ensureReady(signal),
|
|
49
|
+
manager: () => manager,
|
|
50
|
+
});
|
|
51
|
+
const context = {
|
|
52
|
+
workingDir: directory,
|
|
53
|
+
conversationId: "conv-smoke",
|
|
54
|
+
sourceActorPrincipalId: "user-123",
|
|
55
|
+
trustClass: "guardian" as const,
|
|
56
|
+
};
|
|
57
|
+
let submissions = 0;
|
|
58
|
+
let navigations = 0;
|
|
59
|
+
const page = Bun.serve({
|
|
60
|
+
hostname: "127.0.0.1",
|
|
61
|
+
port: 0,
|
|
62
|
+
fetch(request) {
|
|
63
|
+
if (new URL(request.url).pathname === "/save") {
|
|
64
|
+
submissions++;
|
|
65
|
+
return Response.json({ ok: true });
|
|
66
|
+
}
|
|
67
|
+
if (new URL(request.url).pathname === "/") {
|
|
68
|
+
navigations++;
|
|
69
|
+
}
|
|
70
|
+
return new Response(
|
|
71
|
+
`<!doctype html><html><head><title>Desktop browser CLI</title><style>body{font:24px sans-serif;padding:70px;background:#f3f4f8}input,button{font:inherit;padding:12px;margin:16px}#result{color:#5140bd}#colors{position:fixed;left:0;top:0;display:flex}#colors span{width:100px;height:50px}</style></head><body><div id="colors"><span style="background:#ff0000"></span><span style="background:#00ff00"></span><span style="background:#0000ff"></span></div><h1>Streamed desktop browser</h1><label>Example text<input id="text"></label><button id="save" onclick="fetch('/save');document.getElementById('result').textContent='Saved '+document.getElementById('text').value">Save</button><p id="result"></p><button id="open-dialog" onclick="document.querySelector('dialog').showModal()">Open dialog</button><dialog style="background:white"><button id="inside" onclick="this.textContent='Modal clicked'">Inside dialog</button></dialog></body></html>`,
|
|
72
|
+
{
|
|
73
|
+
headers: {
|
|
74
|
+
"content-type": "text/html",
|
|
75
|
+
"content-security-policy":
|
|
76
|
+
"require-trusted-types-for 'script'; trusted-types 'none'",
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
);
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
await mkdir(process.env.ASSISTANT_IPC_SOCKET_DIR, { recursive: true });
|
|
83
|
+
let setupNotifications = 0;
|
|
84
|
+
const ipc = createServer((socket) => {
|
|
85
|
+
const reader = new IpcFrameReader((request) => {
|
|
86
|
+
void (async () => {
|
|
87
|
+
try {
|
|
88
|
+
if (request.method === "/events/publish") {
|
|
89
|
+
setupNotifications++;
|
|
90
|
+
writeMessage(socket, { id: request.id, result: { ok: true } });
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const body = (
|
|
94
|
+
request.params as {
|
|
95
|
+
body: {
|
|
96
|
+
operation: BrowserOperation;
|
|
97
|
+
input: Record<string, unknown>;
|
|
98
|
+
desktop: boolean;
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
).body;
|
|
102
|
+
assert.equal(request.method, "browser_execute");
|
|
103
|
+
assert.equal(body.desktop, true);
|
|
104
|
+
const result = await control.runBrowser(
|
|
105
|
+
context,
|
|
106
|
+
async (signal) => {
|
|
107
|
+
const cdpClient = await manager.browser.client(
|
|
108
|
+
context.conversationId,
|
|
109
|
+
signal,
|
|
110
|
+
);
|
|
111
|
+
return executeBrowserOperation(body.operation, body.input, {
|
|
112
|
+
...context,
|
|
113
|
+
signal,
|
|
114
|
+
cdpClient,
|
|
115
|
+
});
|
|
116
|
+
},
|
|
117
|
+
body.operation === "detach",
|
|
118
|
+
);
|
|
119
|
+
const screenshots = (result.contentBlocks ?? []).flatMap((block) =>
|
|
120
|
+
block.type === "image" && block.source.type === "base64"
|
|
121
|
+
? [{ mediaType: block.source.media_type, data: block.source.data }]
|
|
122
|
+
: [],
|
|
123
|
+
);
|
|
124
|
+
writeMessage(socket, {
|
|
125
|
+
id: request.id,
|
|
126
|
+
result: { ...result, screenshots },
|
|
127
|
+
});
|
|
128
|
+
} catch (error) {
|
|
129
|
+
writeMessage(socket, { id: request.id, error: String(error) });
|
|
130
|
+
}
|
|
131
|
+
})();
|
|
132
|
+
});
|
|
133
|
+
socket.on("data", (chunk) => reader.push(chunk));
|
|
134
|
+
});
|
|
135
|
+
await new Promise<void>((resolve, reject) => {
|
|
136
|
+
ipc.once("error", reject);
|
|
137
|
+
ipc.listen(getAssistantSocketPath(), resolve);
|
|
138
|
+
});
|
|
139
|
+
async function cli(...args: string[]) {
|
|
140
|
+
const program = new Command();
|
|
141
|
+
program.exitOverride();
|
|
142
|
+
registerBrowserCommand(program);
|
|
143
|
+
const chunks: string[] = [];
|
|
144
|
+
const original = process.stdout.write;
|
|
145
|
+
process.stdout.write = ((chunk: unknown) => {
|
|
146
|
+
chunks.push(String(chunk));
|
|
147
|
+
return true;
|
|
148
|
+
}) as typeof original;
|
|
149
|
+
try {
|
|
150
|
+
await program.parseAsync([
|
|
151
|
+
"bun",
|
|
152
|
+
"assistant",
|
|
153
|
+
"browser",
|
|
154
|
+
"--virtual-desktop",
|
|
155
|
+
"--json",
|
|
156
|
+
...args,
|
|
157
|
+
]);
|
|
158
|
+
} finally {
|
|
159
|
+
process.stdout.write = original;
|
|
160
|
+
}
|
|
161
|
+
const result = JSON.parse(chunks.join("")) as {
|
|
162
|
+
ok: boolean;
|
|
163
|
+
content: string;
|
|
164
|
+
error?: string;
|
|
165
|
+
};
|
|
166
|
+
assert.equal(result.ok, true, JSON.stringify(result));
|
|
167
|
+
return result.content;
|
|
168
|
+
}
|
|
169
|
+
function ref(snapshot: string, name: string): string {
|
|
170
|
+
const line = snapshot
|
|
171
|
+
.split("\n")
|
|
172
|
+
.find((value) => value.includes(name) && /\be\d+\b/.test(value));
|
|
173
|
+
assert(line, `Missing ${name}: ${snapshot}`);
|
|
174
|
+
return /\be\d+\b/.exec(line)![0];
|
|
175
|
+
}
|
|
176
|
+
try {
|
|
177
|
+
await cli(
|
|
178
|
+
"navigate",
|
|
179
|
+
"--url",
|
|
180
|
+
`http://127.0.0.1:${page.port}`,
|
|
181
|
+
"--allow-private-network",
|
|
182
|
+
);
|
|
183
|
+
assert.equal(navigations, 1);
|
|
184
|
+
if (coldInstall) {
|
|
185
|
+
assert.equal(desktopDependencyInstaller.getStatus().state, "ready");
|
|
186
|
+
assert(
|
|
187
|
+
setupNotifications >= 3,
|
|
188
|
+
"Installation progress must reach the viewer",
|
|
189
|
+
);
|
|
190
|
+
console.error(
|
|
191
|
+
"PASS: one CLI navigate installed desktop and Chrome from scratch, then loaded the requested page once",
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
const snapshot = await cli("snapshot");
|
|
195
|
+
await cli(
|
|
196
|
+
"type",
|
|
197
|
+
"--element-id",
|
|
198
|
+
ref(snapshot, "Example text"),
|
|
199
|
+
"--text",
|
|
200
|
+
"Hello café 世界",
|
|
201
|
+
"--clear-first",
|
|
202
|
+
);
|
|
203
|
+
await cli("click", "--element-id", ref(snapshot, "Save"));
|
|
204
|
+
const saved = await cli("extract");
|
|
205
|
+
assert.match(saved, /Saved Hello café 世界/);
|
|
206
|
+
assert.equal(submissions, 1);
|
|
207
|
+
const screenshotPath = join(tmpdir(), "desktop-browser-cli-page.jpg");
|
|
208
|
+
await cli("screenshot", "--output", screenshotPath);
|
|
209
|
+
const { data: pixels, info } = await sharp(screenshotPath)
|
|
210
|
+
.removeAlpha()
|
|
211
|
+
.raw()
|
|
212
|
+
.toBuffer({ resolveWithObject: true });
|
|
213
|
+
for (let channel = 0; channel < 3; channel++) {
|
|
214
|
+
const offset = (25 * info.width + 50 + channel * 100) * info.channels;
|
|
215
|
+
for (let component = 0; component < 3; component++) {
|
|
216
|
+
const expected = component === channel ? 255 : 0;
|
|
217
|
+
assert(
|
|
218
|
+
Math.abs(pixels[offset + component] - expected) < 10,
|
|
219
|
+
`Incorrect screenshot color for swatch ${channel}, channel ${component}`,
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
async function assertCursorPainted(expected: boolean) {
|
|
224
|
+
await control.runBrowser(context, async (signal) => {
|
|
225
|
+
const cdp = await manager.browser.client(context.conversationId, signal);
|
|
226
|
+
const screenshot = await cdp.send<{ data: string }>(
|
|
227
|
+
"Page.captureScreenshot",
|
|
228
|
+
{ format: "png" },
|
|
229
|
+
);
|
|
230
|
+
const { data, info } = await sharp(Buffer.from(screenshot.data, "base64"))
|
|
231
|
+
.removeAlpha()
|
|
232
|
+
.raw()
|
|
233
|
+
.toBuffer({ resolveWithObject: true });
|
|
234
|
+
let purple = 0;
|
|
235
|
+
for (let i = 0; i < data.length; i += info.channels) {
|
|
236
|
+
if (
|
|
237
|
+
Math.abs(data[i]! - 112) < 5 &&
|
|
238
|
+
Math.abs(data[i + 1]! - 87) < 5 &&
|
|
239
|
+
Math.abs(data[i + 2]! - 255) < 5
|
|
240
|
+
) {
|
|
241
|
+
purple++;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
assert.equal(purple > 40, expected, `Cursor pixels: ${purple}`);
|
|
245
|
+
return { content: "verified", isError: false };
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
await assertCursorPainted(true);
|
|
249
|
+
await cli("click", "--selector", "#open-dialog");
|
|
250
|
+
await assertCursorPainted(true);
|
|
251
|
+
await cli("hover", "--selector", "#inside");
|
|
252
|
+
await assertCursorPainted(true);
|
|
253
|
+
await cli("click", "--selector", "#inside");
|
|
254
|
+
assert.match(await cli("extract"), /Modal clicked/);
|
|
255
|
+
for (const key of ["Enter", "Space"]) {
|
|
256
|
+
await cli("press-key", "--key", "Escape");
|
|
257
|
+
await cli("press-key", "--key", key, "--selector", "#open-dialog");
|
|
258
|
+
await assertCursorPainted(true);
|
|
259
|
+
}
|
|
260
|
+
await cli(
|
|
261
|
+
"navigate",
|
|
262
|
+
"--url",
|
|
263
|
+
`http://127.0.0.1:${page.port}/next`,
|
|
264
|
+
"--allow-private-network",
|
|
265
|
+
);
|
|
266
|
+
await assertCursorPainted(true);
|
|
267
|
+
await cli("detach");
|
|
268
|
+
await assertCursorPainted(false);
|
|
269
|
+
await cli("detach");
|
|
270
|
+
console.log(
|
|
271
|
+
"PASS: real browser CLI over IPC, shared AX snapshot, Unicode typing, one click submission, RGB page screenshot, cursor pixels above dialogs and after navigation, detach cleanup",
|
|
272
|
+
);
|
|
273
|
+
} finally {
|
|
274
|
+
await cli("detach");
|
|
275
|
+
await manager.destroy();
|
|
276
|
+
await new Promise<void>((resolve) => ipc.close(() => resolve()));
|
|
277
|
+
page.stop(true);
|
|
278
|
+
await rm(directory, { recursive: true, force: true });
|
|
279
|
+
}
|
|
@@ -5,6 +5,7 @@ import { SYNC_TAGS } from "../daemon/message-types/sync.js";
|
|
|
5
5
|
import { assistantEventHub } from "../runtime/assistant-event-hub.js";
|
|
6
6
|
import {
|
|
7
7
|
publishConfigChanged,
|
|
8
|
+
publishMcpChanged,
|
|
8
9
|
publishSchedulesChanged,
|
|
9
10
|
publishSoundsConfigUpdated,
|
|
10
11
|
} from "../runtime/sync/resource-sync-events.js";
|
|
@@ -94,4 +95,26 @@ describe("config and sounds sync events", () => {
|
|
|
94
95
|
subscription.dispose();
|
|
95
96
|
}
|
|
96
97
|
});
|
|
98
|
+
|
|
99
|
+
test("MCP changes emit a sync tag without a client origin", async () => {
|
|
100
|
+
const received: AssistantEventEnvelope[] = [];
|
|
101
|
+
const subscription = assistantEventHub.subscribe({
|
|
102
|
+
type: "process",
|
|
103
|
+
callback: (event) => {
|
|
104
|
+
received.push(event);
|
|
105
|
+
},
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
try {
|
|
109
|
+
publishMcpChanged();
|
|
110
|
+
await waitFor(() => received.length === 1);
|
|
111
|
+
|
|
112
|
+
expect(received[0].message).toEqual({
|
|
113
|
+
type: "sync_changed",
|
|
114
|
+
tags: [SYNC_TAGS.mcpList],
|
|
115
|
+
});
|
|
116
|
+
} finally {
|
|
117
|
+
subscription.dispose();
|
|
118
|
+
}
|
|
119
|
+
});
|
|
97
120
|
});
|
|
@@ -1490,6 +1490,7 @@ describe("classifyConversationError", () => {
|
|
|
1490
1490
|
"subagent_aborted",
|
|
1491
1491
|
"signal_cancel",
|
|
1492
1492
|
"voice_session_aborted",
|
|
1493
|
+
"voice_progress_narration_timeout",
|
|
1493
1494
|
];
|
|
1494
1495
|
|
|
1495
1496
|
for (const kind of taggedKinds) {
|
|
@@ -1727,7 +1728,9 @@ describe("ConnectionResolutionError classification", () => {
|
|
|
1727
1728
|
expect(result.userMessage).toContain("qwen/qwen3-8b");
|
|
1728
1729
|
expect(result.userMessage).toContain("Vellum GPU route");
|
|
1729
1730
|
expect(result.userMessage).toContain('profile "steer"');
|
|
1730
|
-
expect(result.userMessage).toContain(
|
|
1731
|
+
expect(result.userMessage).toContain(
|
|
1732
|
+
"was not sent through another provider",
|
|
1733
|
+
);
|
|
1731
1734
|
});
|
|
1732
1735
|
|
|
1733
1736
|
it("classifies missing_credential naming the connection and fix", () => {
|
|
@@ -607,7 +607,9 @@ describe("injectChannelCapabilityContext", () => {
|
|
|
607
607
|
"ui_show, ui_update, and ui_dismiss persist conversation content",
|
|
608
608
|
);
|
|
609
609
|
expect(text).not.toContain("Only use ui_show/ui_update");
|
|
610
|
-
expect(text).not.toContain(
|
|
610
|
+
expect(text).not.toContain(
|
|
611
|
+
"Do NOT use ui_show, ui_update, or app_create",
|
|
612
|
+
);
|
|
611
613
|
expect(text).not.toContain(
|
|
612
614
|
"Present information as well-formatted text instead of dynamic UI.",
|
|
613
615
|
);
|
|
@@ -898,12 +900,12 @@ describe("trust-gating via channel capabilities", () => {
|
|
|
898
900
|
|
|
899
901
|
const result = injectChannelCapabilityContext(message, caps);
|
|
900
902
|
|
|
901
|
-
// macOS clients
|
|
903
|
+
// macOS clients get app-scripting guidance injected
|
|
902
904
|
expect(result).not.toBe(message);
|
|
903
905
|
const injected = (result.content[0] as { type: "text"; text: string }).text;
|
|
904
906
|
expect(injected).toContain("client_os: macos");
|
|
905
|
-
expect(injected).toContain("
|
|
906
|
-
expect(injected).toContain("host_bash");
|
|
907
|
+
expect(injected).toContain("drive apps with the computer-use skill");
|
|
908
|
+
expect(injected).toContain("`host_bash` is for shell commands");
|
|
907
909
|
// No channel constraints — full desktop capabilities
|
|
908
910
|
expect(injected).not.toContain("CHANNEL CONSTRAINTS");
|
|
909
911
|
});
|
|
@@ -39,6 +39,20 @@ function writePackageJson(dir: string, pkg: Record<string, unknown>): void {
|
|
|
39
39
|
writeFileSync(join(dir, "package.json"), JSON.stringify(pkg, null, 2));
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
+
function writePluginJson(dir: string, manifest: Record<string, unknown>): void {
|
|
43
|
+
writeFileSync(
|
|
44
|
+
join(dir, "plugin.json"),
|
|
45
|
+
JSON.stringify(
|
|
46
|
+
{
|
|
47
|
+
$schema: "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
48
|
+
...manifest,
|
|
49
|
+
},
|
|
50
|
+
null,
|
|
51
|
+
2,
|
|
52
|
+
),
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
42
56
|
function writeSurfaceFile(dir: string, relPath: string, body: string): void {
|
|
43
57
|
const parts = relPath.split("/");
|
|
44
58
|
parts.pop();
|
|
@@ -104,6 +118,31 @@ describe("loadExternalPlugin — manifest", () => {
|
|
|
104
118
|
expect(registered).toBeDefined();
|
|
105
119
|
expect(registered?.manifest.version).toBe("0.0.0");
|
|
106
120
|
});
|
|
121
|
+
|
|
122
|
+
test("loads a standard plugin.json when package.json is absent", async () => {
|
|
123
|
+
const dir = freshPluginDir("standard-only");
|
|
124
|
+
writePluginJson(dir, { name: "standard-only", version: "2.1.0" });
|
|
125
|
+
|
|
126
|
+
await loadExternalPlugin(dir);
|
|
127
|
+
|
|
128
|
+
const registered = getRegisteredPlugins().find(
|
|
129
|
+
(plugin) => plugin.manifest.name === "standard-only",
|
|
130
|
+
);
|
|
131
|
+
expect(registered?.manifest.version).toBe("2.1.0");
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("keeps package.json authoritative when plugin.json is also present", async () => {
|
|
135
|
+
const dir = freshPluginDir("mixed-manifests");
|
|
136
|
+
writePackageJson(dir, { name: "legacy-name", version: "1.2.3" });
|
|
137
|
+
writeFileSync(join(dir, "plugin.json"), "{ foreign and malformed }");
|
|
138
|
+
|
|
139
|
+
await loadExternalPlugin(dir);
|
|
140
|
+
|
|
141
|
+
const registered = getRegisteredPlugins().find(
|
|
142
|
+
(plugin) => plugin.manifest.name === "mixed-manifests",
|
|
143
|
+
);
|
|
144
|
+
expect(registered?.manifest.version).toBe("1.2.3");
|
|
145
|
+
});
|
|
107
146
|
});
|
|
108
147
|
|
|
109
148
|
describe("credentialKeyPatterns manifest field", () => {
|
|
@@ -277,6 +316,19 @@ describe("loadExternalPlugin — plugin-api peerDependency", () => {
|
|
|
277
316
|
test("malformed package.json is logged and skipped (registry untouched)", async () => {
|
|
278
317
|
const dir = freshPluginDir("malformed-pkg");
|
|
279
318
|
writeFileSync(join(dir, "package.json"), "{ this is not json");
|
|
319
|
+
writePluginJson(dir, { name: "valid-standard-fallback" });
|
|
320
|
+
|
|
321
|
+
await loadExternalPlugin(dir);
|
|
322
|
+
|
|
323
|
+
expect(registeredNames()).toHaveLength(0);
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
test("malformed selected plugin.json is logged and skipped", async () => {
|
|
327
|
+
const dir = freshPluginDir("malformed-standard");
|
|
328
|
+
writeFileSync(
|
|
329
|
+
join(dir, "plugin.json"),
|
|
330
|
+
JSON.stringify({ name: "missing-schema" }),
|
|
331
|
+
);
|
|
280
332
|
|
|
281
333
|
await loadExternalPlugin(dir);
|
|
282
334
|
|
|
@@ -41,6 +41,16 @@ function makeFakeCdp(
|
|
|
41
41
|
dispose() {
|
|
42
42
|
_cdpDisposed = true;
|
|
43
43
|
},
|
|
44
|
+
setCdpSessionId() {},
|
|
45
|
+
async listTabs() {
|
|
46
|
+
return [];
|
|
47
|
+
},
|
|
48
|
+
async selectTab(tabId: number) {
|
|
49
|
+
return { tabId };
|
|
50
|
+
},
|
|
51
|
+
async closeTab(tabId: number) {
|
|
52
|
+
return { tabId, closed: true };
|
|
53
|
+
},
|
|
44
54
|
};
|
|
45
55
|
}
|
|
46
56
|
|
|
@@ -142,6 +152,8 @@ mock.module("../tools/network/url-safety.js", () => ({
|
|
|
142
152
|
|
|
143
153
|
import {
|
|
144
154
|
executeBrowserClick,
|
|
155
|
+
executeBrowserClose,
|
|
156
|
+
executeBrowserDetach,
|
|
145
157
|
executeBrowserNavigate,
|
|
146
158
|
executeBrowserScreenshot,
|
|
147
159
|
executeBrowserSnapshot,
|
|
@@ -687,6 +699,42 @@ describe("browser_mode wiring through tool execution", () => {
|
|
|
687
699
|
|
|
688
700
|
// ── Per-conversation sticky backend kind ─────────────────────────
|
|
689
701
|
|
|
702
|
+
for (const release of [executeBrowserDetach, executeBrowserClose]) {
|
|
703
|
+
test.each(["local", "extension", "cdp-inspect"])(
|
|
704
|
+
`${release.name} on desktop preserves the normal %s session's backend`,
|
|
705
|
+
async (mode) => {
|
|
706
|
+
const first = await executeBrowserSnapshot({ browser_mode: mode }, ctx);
|
|
707
|
+
expect(first.isError).toBe(false);
|
|
708
|
+
|
|
709
|
+
factoryModeCalls.length = 0;
|
|
710
|
+
const desktop = await release(
|
|
711
|
+
{},
|
|
712
|
+
{
|
|
713
|
+
...ctx,
|
|
714
|
+
cdpClient: makeFakeCdp(
|
|
715
|
+
"cdp-inspect",
|
|
716
|
+
`desktop-browser:${ctx.conversationId}`,
|
|
717
|
+
),
|
|
718
|
+
},
|
|
719
|
+
);
|
|
720
|
+
expect(desktop.isError).toBe(false);
|
|
721
|
+
expect(factoryModeCalls).toEqual([]);
|
|
722
|
+
|
|
723
|
+
const resumed = await executeBrowserSnapshot({}, ctx);
|
|
724
|
+
expect(resumed.isError).toBe(false);
|
|
725
|
+
expect(factoryModeCalls).toEqual([mode]);
|
|
726
|
+
},
|
|
727
|
+
);
|
|
728
|
+
|
|
729
|
+
test(`${release.name} on the normal browser still resets automatic selection`, async () => {
|
|
730
|
+
await executeBrowserSnapshot({ browser_mode: "extension" }, ctx);
|
|
731
|
+
expect((await release({}, ctx)).isError).toBe(false);
|
|
732
|
+
factoryModeCalls.length = 0;
|
|
733
|
+
expect((await executeBrowserSnapshot({}, ctx)).isError).toBe(false);
|
|
734
|
+
expect(factoryModeCalls).toEqual(["auto"]);
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
|
|
690
738
|
test("auto-mode call after an explicit pin sticks to the pinned kind", async () => {
|
|
691
739
|
const first = await executeBrowserNavigate(
|
|
692
740
|
{ url: "https://example.com", browser_mode: "local" },
|