@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/desktop-browser-cli.md +33 -0
- package/openapi.yaml +17 -5
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +279 -0
- package/src/__tests__/config-schema.test.ts +1 -1
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/host-cu-proxy.test.ts +116 -18
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/mcp-auth-routes.test.ts +99 -0
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-health-check.test.ts +32 -32
- package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/skills.test.ts +10 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/acp/session-snapshot.ts +260 -0
- package/src/apps/app-store.ts +6 -5
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/calls/__tests__/progress-narration.test.ts +23 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
- package/src/calls/progress-narration.ts +15 -2
- package/src/calls/voice-control-protocol.ts +34 -4
- package/src/calls/voice-session-bridge.ts +23 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/plugins.ts +16 -5
- package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +13 -0
- package/src/cli/lib/install-from-github.ts +17 -18
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-details.ts +21 -8
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/bundled-skills/acp/SKILL.md +1 -1
- package/src/config/bundled-skills/acp/TOOLS.json +2 -2
- package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
- package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
- package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
- package/src/config/feature-flag-registry.json +2 -2
- package/src/config/schemas/__tests__/voice.test.ts +2 -2
- package/src/config/schemas/mcp.ts +9 -7
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/voice.ts +2 -2
- package/src/config/skills.ts +13 -21
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-runtime-assembly.ts +1 -1
- package/src/daemon/conversation-tool-setup.ts +31 -29
- package/src/daemon/host-cu-proxy.ts +51 -26
- package/src/daemon/host-proxy-preactivation.ts +2 -0
- package/src/daemon/mcp-reload-service.ts +7 -0
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/providers-setup.ts +4 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-automation-lease.test.ts +208 -0
- package/src/desktop/desktop-automation-lease.ts +270 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +58 -8
- package/src/desktop/desktop-dependencies.test.ts +29 -0
- package/src/desktop/desktop-dependencies.ts +26 -5
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.ts +14 -1
- package/src/desktop/desktop-session-manager.test.ts +86 -3
- package/src/desktop/desktop-session-manager.ts +144 -24
- package/src/desktop/desktop-stream-bridge.ts +1 -1
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/i18n/__tests__/i18n.test.ts +9 -5
- package/src/i18n/messages.ts +13 -4
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/protocol.test.ts +11 -3
- package/src/live-voice/__tests__/session-controls.test.ts +34 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-session.ts +3 -0
- package/src/live-voice/protocol.ts +10 -1
- package/src/live-voice/session-controls.ts +28 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +11 -0
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +42 -15
- package/src/mcp/manager.ts +61 -8
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/notifications/__tests__/decision-engine.test.ts +175 -0
- package/src/notifications/decision-engine.ts +43 -8
- package/src/oauth/seed-providers.ts +11 -0
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +9 -15
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
- package/src/providers/inference/endpoint-probe.ts +27 -12
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +167 -0
- package/src/providers/opencode/client.ts +73 -4
- package/src/runtime/AGENTS.md +4 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
- package/src/runtime/http-server.ts +6 -6
- package/src/runtime/routes/__tests__/acp-routes.test.ts +19 -0
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
- package/src/runtime/routes/acp-routes.ts +17 -161
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
- package/src/runtime/routes/desktop-setup-routes.ts +7 -5
- package/src/runtime/routes/mcp-auth-routes.ts +106 -94
- package/src/runtime/routes/plugins-routes.ts +14 -1
- package/src/runtime/sync/resource-sync-events.ts +4 -0
- package/src/tools/acp/status.test.ts +276 -21
- package/src/tools/acp/status.ts +98 -32
- package/src/tools/browser/browser-execution.ts +44 -13
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/computer-use/definitions.ts +2 -3
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/plugin-manifest.ts +203 -0
- package/src/desktop/desktop-feature.test.ts +0 -18
- package/src/desktop/desktop-feature.ts +0 -19
package/src/tools/acp/status.ts
CHANGED
|
@@ -1,13 +1,37 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
3
|
+
import { resolvedClaudeCredentialDigest } from "../../acp/prepare-agent-env.js";
|
|
4
|
+
import {
|
|
5
|
+
type AcpSessionSnapshot,
|
|
6
|
+
getAcpSessionSnapshot,
|
|
7
|
+
listAcpSessionSnapshots,
|
|
8
|
+
withCurrentAuthMarkers,
|
|
9
|
+
} from "../../acp/session-snapshot.js";
|
|
5
10
|
import {
|
|
6
11
|
invalidToolInputResult,
|
|
7
12
|
nullAsOmitted,
|
|
8
13
|
} from "../shared/zod-tool-schema.js";
|
|
9
14
|
import type { ToolContext, ToolExecutionResult } from "../types.js";
|
|
10
15
|
|
|
16
|
+
const STATUS_LIST_LIMIT = 50;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Keep every live session visible, then fill remaining capacity from durable
|
|
20
|
+
* history. An older active run must not be displaced by newer historical rows.
|
|
21
|
+
*/
|
|
22
|
+
function selectStatusSnapshots(
|
|
23
|
+
sessions: AcpSessionSnapshot[],
|
|
24
|
+
): AcpSessionSnapshot[] {
|
|
25
|
+
const live = sessions.filter((session) => session.source === "live");
|
|
26
|
+
const history = sessions
|
|
27
|
+
.filter((session) => session.source === "history")
|
|
28
|
+
.sort((a, b) => b.startedAt - a.startedAt);
|
|
29
|
+
const remaining = Math.max(0, STATUS_LIST_LIMIT - live.length);
|
|
30
|
+
return [...live, ...history.slice(0, remaining)].sort(
|
|
31
|
+
(a, b) => b.startedAt - a.startedAt,
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
|
|
11
35
|
/**
|
|
12
36
|
* Model-input schema, `safeParse`d at the top of {@link executeAcpStatus}.
|
|
13
37
|
* Same in-tool pattern and TOOLS.json drift guard as the other bundled-skill
|
|
@@ -24,33 +48,48 @@ export const acpStatusInputSchema = z.looseObject({
|
|
|
24
48
|
* `AcpSessionState` for those surfaces should not reach this one by default.
|
|
25
49
|
* `model` stays: which model a session is running on is answerable.
|
|
26
50
|
*/
|
|
27
|
-
function projectSession(
|
|
51
|
+
function projectSession(snapshot: AcpSessionSnapshot) {
|
|
52
|
+
const fromHistory = snapshot.source === "history";
|
|
53
|
+
const idle =
|
|
54
|
+
fromHistory &&
|
|
55
|
+
snapshot.status === "completed" &&
|
|
56
|
+
snapshot.stopReason !== "cancelled" &&
|
|
57
|
+
snapshot.resumable;
|
|
58
|
+
const latestUsage =
|
|
59
|
+
snapshot.usedTokens !== undefined && snapshot.contextSize !== undefined
|
|
60
|
+
? {
|
|
61
|
+
usedTokens: snapshot.usedTokens,
|
|
62
|
+
contextSize: snapshot.contextSize,
|
|
63
|
+
costAmount: snapshot.costAmount,
|
|
64
|
+
costCurrency: snapshot.costCurrency,
|
|
65
|
+
inputTokens: snapshot.inputTokens,
|
|
66
|
+
outputTokens: snapshot.outputTokens,
|
|
67
|
+
}
|
|
68
|
+
: undefined;
|
|
28
69
|
return {
|
|
29
|
-
id:
|
|
30
|
-
agentId:
|
|
31
|
-
acpSessionId:
|
|
32
|
-
parentConversationId:
|
|
33
|
-
status:
|
|
34
|
-
startedAt:
|
|
35
|
-
completedAt:
|
|
36
|
-
error:
|
|
37
|
-
stopReason:
|
|
38
|
-
task:
|
|
39
|
-
parentToolUseId:
|
|
40
|
-
authErrorCode:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
70
|
+
id: snapshot.id,
|
|
71
|
+
agentId: snapshot.agentId,
|
|
72
|
+
acpSessionId: snapshot.acpSessionId,
|
|
73
|
+
parentConversationId: snapshot.parentConversationId,
|
|
74
|
+
status: idle ? "idle" : snapshot.status,
|
|
75
|
+
startedAt: snapshot.startedAt,
|
|
76
|
+
completedAt: snapshot.completedAt ?? undefined,
|
|
77
|
+
error: snapshot.error ?? undefined,
|
|
78
|
+
stopReason: snapshot.stopReason ?? undefined,
|
|
79
|
+
task: snapshot.task,
|
|
80
|
+
parentToolUseId: snapshot.parentToolUseId,
|
|
81
|
+
authErrorCode: snapshot.authErrorCode,
|
|
82
|
+
latestUsage,
|
|
83
|
+
model: snapshot.model,
|
|
84
|
+
...(fromHistory
|
|
85
|
+
? {
|
|
86
|
+
resumable: snapshot.resumable,
|
|
87
|
+
lastRunStatus: snapshot.status,
|
|
88
|
+
}
|
|
89
|
+
: {}),
|
|
44
90
|
};
|
|
45
91
|
}
|
|
46
92
|
|
|
47
|
-
/** Projects either shape `getStatus` answers with. */
|
|
48
|
-
function projectStatus(status: AcpSessionState | AcpSessionState[]): unknown {
|
|
49
|
-
return Array.isArray(status)
|
|
50
|
-
? status.map(projectSession)
|
|
51
|
-
: projectSession(status);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
93
|
export async function executeAcpStatus(
|
|
55
94
|
input: Record<string, unknown>,
|
|
56
95
|
_context: ToolContext,
|
|
@@ -60,24 +99,51 @@ export async function executeAcpStatus(
|
|
|
60
99
|
return invalidToolInputResult("acp_status", parsedInput.error);
|
|
61
100
|
}
|
|
62
101
|
const acpSessionId = parsedInput.data.acp_session_id;
|
|
63
|
-
const manager = getAcpSessionManager();
|
|
64
|
-
|
|
65
102
|
try {
|
|
66
103
|
if (acpSessionId) {
|
|
104
|
+
const snapshot = getAcpSessionSnapshot(acpSessionId, {
|
|
105
|
+
includeEventLog: false,
|
|
106
|
+
});
|
|
107
|
+
if (!snapshot) {
|
|
108
|
+
return {
|
|
109
|
+
content: `ACP session "${acpSessionId}" not found`,
|
|
110
|
+
isError: true,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
const judged = (
|
|
114
|
+
await withCurrentAuthMarkers(
|
|
115
|
+
[snapshot],
|
|
116
|
+
resolvedClaudeCredentialDigest,
|
|
117
|
+
)
|
|
118
|
+
)[0];
|
|
119
|
+
if (!judged) {
|
|
120
|
+
return {
|
|
121
|
+
content: `ACP session "${acpSessionId}" not found`,
|
|
122
|
+
isError: true,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
67
125
|
return {
|
|
68
|
-
content: JSON.stringify(
|
|
126
|
+
content: JSON.stringify(projectSession(judged)),
|
|
69
127
|
isError: false,
|
|
70
128
|
};
|
|
71
129
|
}
|
|
72
130
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
131
|
+
const snapshots = selectStatusSnapshots(
|
|
132
|
+
listAcpSessionSnapshots({
|
|
133
|
+
limit: STATUS_LIST_LIMIT,
|
|
134
|
+
includeEventLog: false,
|
|
135
|
+
}).sessions,
|
|
136
|
+
);
|
|
137
|
+
if (snapshots.length === 0) {
|
|
76
138
|
return { content: "No ACP sessions found.", isError: false };
|
|
77
139
|
}
|
|
78
140
|
|
|
141
|
+
const judged = await withCurrentAuthMarkers(
|
|
142
|
+
snapshots,
|
|
143
|
+
resolvedClaudeCredentialDigest,
|
|
144
|
+
);
|
|
79
145
|
return {
|
|
80
|
-
content: JSON.stringify(
|
|
146
|
+
content: JSON.stringify(judged.map(projectSession)),
|
|
81
147
|
isError: false,
|
|
82
148
|
};
|
|
83
149
|
} catch (err) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { optimizeImageForTransport } from "../../agent/image-optimize.js";
|
|
2
|
+
import type { BrowserOperationContext as ToolContext } from "../../browser/types.js";
|
|
2
3
|
import { getConfig } from "../../config/loader.js";
|
|
3
4
|
import { HostBrowserProxy } from "../../daemon/host-browser-proxy.js";
|
|
4
5
|
import type { ImageContent } from "../../providers/types.js";
|
|
@@ -21,7 +22,7 @@ import {
|
|
|
21
22
|
sanitizeUrlForOutput,
|
|
22
23
|
sanitizeUrlStringForOutput,
|
|
23
24
|
} from "../network/url-safety.js";
|
|
24
|
-
import type {
|
|
25
|
+
import type { ToolExecutionResult } from "../types.js";
|
|
25
26
|
import {
|
|
26
27
|
type AuthChallenge,
|
|
27
28
|
detectAuthChallenge,
|
|
@@ -515,6 +516,9 @@ async function acquireCdpClientWithMode(
|
|
|
515
516
|
}
|
|
516
517
|
| { cdp?: never; browserMode?: never; errorResult: ToolExecutionResult }
|
|
517
518
|
> {
|
|
519
|
+
if (context.cdpClient) {
|
|
520
|
+
return { cdp: context.cdpClient, browserMode: "cdp-inspect" };
|
|
521
|
+
}
|
|
518
522
|
const modeResult = parseBrowserMode(input);
|
|
519
523
|
if (!modeResult.ok) {
|
|
520
524
|
return {
|
|
@@ -746,6 +750,17 @@ export type ResolvedElement =
|
|
|
746
750
|
| { kind: "backend"; backendNodeId: number; eid: string }
|
|
747
751
|
| { kind: "selector"; selector: string };
|
|
748
752
|
|
|
753
|
+
function snapshotConversationId(context: ToolContext): string {
|
|
754
|
+
return context.cdpClient?.conversationId ?? context.conversationId;
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
function clearBrowserSessionState(context: ToolContext): void {
|
|
758
|
+
browserManager.clearSnapshotBackendNodeMap(snapshotConversationId(context));
|
|
759
|
+
if (!context.cdpClient) {
|
|
760
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
749
764
|
/**
|
|
750
765
|
* Resolve an element reference (either `element_id` from a prior
|
|
751
766
|
* snapshot or a raw `selector`) for CDP-native tools. Returns a
|
|
@@ -872,6 +887,9 @@ export async function executeBrowserNavigate(
|
|
|
872
887
|
typeof input.target_client_id === "string" && input.target_client_id !== ""
|
|
873
888
|
? input.target_client_id
|
|
874
889
|
: undefined;
|
|
890
|
+
if (context.cdpClient && forceNewTab) {
|
|
891
|
+
await cdp.send("Vellum.createTab", {}, context.signal);
|
|
892
|
+
}
|
|
875
893
|
if (cdp.kind === "extension" && useActiveTab) {
|
|
876
894
|
// Explicit opt-out: target the currently-active tab. Clear any
|
|
877
895
|
// conversation pin and reset the live session so this navigate is
|
|
@@ -1188,7 +1206,7 @@ export async function executeBrowserNavigate(
|
|
|
1188
1206
|
// Navigation changed the page content, so clear stale snapshot
|
|
1189
1207
|
// mappings regardless of backend. The backendNodeId map is shared
|
|
1190
1208
|
// per-conversation state that needs to be invalidated on any nav.
|
|
1191
|
-
browserManager.clearSnapshotBackendNodeMap(context
|
|
1209
|
+
browserManager.clearSnapshotBackendNodeMap(snapshotConversationId(context));
|
|
1192
1210
|
|
|
1193
1211
|
// Auto-dismiss common blocker modals (regulatory notices, cookie
|
|
1194
1212
|
// banners) that aren't exposed in the accessibility tree. Runs
|
|
@@ -1398,7 +1416,7 @@ export async function executeBrowserSnapshot(
|
|
|
1398
1416
|
const { elements, selectorMap: backendNodeMap } = transformAxTree(rawTree);
|
|
1399
1417
|
|
|
1400
1418
|
browserManager.storeSnapshotBackendNodeMap(
|
|
1401
|
-
context
|
|
1419
|
+
snapshotConversationId(context),
|
|
1402
1420
|
backendNodeMap,
|
|
1403
1421
|
);
|
|
1404
1422
|
|
|
@@ -1600,8 +1618,7 @@ export async function executeBrowserDetach(
|
|
|
1600
1618
|
// Vellum.detach round-trip failed (target gone, transport dropped).
|
|
1601
1619
|
// browser_detach is the user's recovery path — leaving a stale
|
|
1602
1620
|
// sticky backend or snapshot map behind would defeat its purpose.
|
|
1603
|
-
|
|
1604
|
-
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
1621
|
+
clearBrowserSessionState(context);
|
|
1605
1622
|
cdp.dispose();
|
|
1606
1623
|
}
|
|
1607
1624
|
}
|
|
@@ -1653,8 +1670,7 @@ export async function executeBrowserClose(
|
|
|
1653
1670
|
// Tolerate detach failures (already detached, tab closed, etc.)
|
|
1654
1671
|
}
|
|
1655
1672
|
}
|
|
1656
|
-
|
|
1657
|
-
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
1673
|
+
clearBrowserSessionState(context);
|
|
1658
1674
|
return {
|
|
1659
1675
|
content:
|
|
1660
1676
|
"Browser session cleared. (Your Chrome tab was not closed — close it yourself if desired.)",
|
|
@@ -1683,7 +1699,10 @@ export async function executeBrowserClick(
|
|
|
1683
1699
|
input: Record<string, unknown>,
|
|
1684
1700
|
context: ToolContext,
|
|
1685
1701
|
): Promise<ToolExecutionResult> {
|
|
1686
|
-
const { resolved, error } = resolveElement(
|
|
1702
|
+
const { resolved, error } = resolveElement(
|
|
1703
|
+
snapshotConversationId(context),
|
|
1704
|
+
input,
|
|
1705
|
+
);
|
|
1687
1706
|
if (error) {
|
|
1688
1707
|
return { content: error, isError: true };
|
|
1689
1708
|
}
|
|
@@ -1796,7 +1815,10 @@ export async function executeBrowserType(
|
|
|
1796
1815
|
input: Record<string, unknown>,
|
|
1797
1816
|
context: ToolContext,
|
|
1798
1817
|
): Promise<ToolExecutionResult> {
|
|
1799
|
-
const { resolved, error } = resolveElement(
|
|
1818
|
+
const { resolved, error } = resolveElement(
|
|
1819
|
+
snapshotConversationId(context),
|
|
1820
|
+
input,
|
|
1821
|
+
);
|
|
1800
1822
|
if (error) {
|
|
1801
1823
|
return { content: error, isError: true };
|
|
1802
1824
|
}
|
|
@@ -1887,7 +1909,7 @@ export async function executeBrowserPressKey(
|
|
|
1887
1909
|
let targetDescription: string | null = null;
|
|
1888
1910
|
let resolved: ResolvedElement | null = null;
|
|
1889
1911
|
if (hasTarget) {
|
|
1890
|
-
const res = resolveElement(context
|
|
1912
|
+
const res = resolveElement(snapshotConversationId(context), input);
|
|
1891
1913
|
if (res.error) {
|
|
1892
1914
|
return { content: res.error, isError: true };
|
|
1893
1915
|
}
|
|
@@ -2025,7 +2047,10 @@ export async function executeBrowserSelectOption(
|
|
|
2025
2047
|
input: Record<string, unknown>,
|
|
2026
2048
|
context: ToolContext,
|
|
2027
2049
|
): Promise<ToolExecutionResult> {
|
|
2028
|
-
const { resolved, error } = resolveElement(
|
|
2050
|
+
const { resolved, error } = resolveElement(
|
|
2051
|
+
snapshotConversationId(context),
|
|
2052
|
+
input,
|
|
2053
|
+
);
|
|
2029
2054
|
if (error) {
|
|
2030
2055
|
return { content: error, isError: true };
|
|
2031
2056
|
}
|
|
@@ -2162,7 +2187,10 @@ export async function executeBrowserHover(
|
|
|
2162
2187
|
input: Record<string, unknown>,
|
|
2163
2188
|
context: ToolContext,
|
|
2164
2189
|
): Promise<ToolExecutionResult> {
|
|
2165
|
-
const { resolved, error } = resolveElement(
|
|
2190
|
+
const { resolved, error } = resolveElement(
|
|
2191
|
+
snapshotConversationId(context),
|
|
2192
|
+
input,
|
|
2193
|
+
);
|
|
2166
2194
|
if (error) {
|
|
2167
2195
|
return { content: error, isError: true };
|
|
2168
2196
|
}
|
|
@@ -2410,7 +2438,10 @@ export async function executeBrowserFillCredential(
|
|
|
2410
2438
|
return { content: "Error: field is required.", isError: true };
|
|
2411
2439
|
}
|
|
2412
2440
|
|
|
2413
|
-
const { resolved, error } = resolveElement(
|
|
2441
|
+
const { resolved, error } = resolveElement(
|
|
2442
|
+
snapshotConversationId(context),
|
|
2443
|
+
input,
|
|
2444
|
+
);
|
|
2414
2445
|
if (error) {
|
|
2415
2446
|
return { content: error, isError: true };
|
|
2416
2447
|
}
|
|
@@ -2804,3 +2804,72 @@ describe("macOS host-browser proxy without extension registry", () => {
|
|
|
2804
2804
|
expect(result).toEqual({ ok: true, via: "extension" });
|
|
2805
2805
|
});
|
|
2806
2806
|
});
|
|
2807
|
+
|
|
2808
|
+
describe("native renderer browser compatibility", () => {
|
|
2809
|
+
beforeEach(() => {
|
|
2810
|
+
setCdpInspectEnabled(false);
|
|
2811
|
+
setDesktopAutoConfig({ enabled: true, cooldownMs: 30_000 });
|
|
2812
|
+
_resetDesktopAutoCooldown();
|
|
2813
|
+
_resetHostBridgeCooldown();
|
|
2814
|
+
mockSingletonProxy = null;
|
|
2815
|
+
});
|
|
2816
|
+
test.each(["macos", "windows"] as const)(
|
|
2817
|
+
"%s web renderer sends CDP to the connected host bridge",
|
|
2818
|
+
async (clientOs) => {
|
|
2819
|
+
mockSingletonProxy = makeMacosBridgeOnlyProxy();
|
|
2820
|
+
const client = getCdpClient(
|
|
2821
|
+
makeContext({
|
|
2822
|
+
conversationId: "conv-native",
|
|
2823
|
+
transportInterface: "web",
|
|
2824
|
+
clientOs,
|
|
2825
|
+
}),
|
|
2826
|
+
);
|
|
2827
|
+
try {
|
|
2828
|
+
expect(await client.send("Page.getFrameTree", {})).toMatchObject({
|
|
2829
|
+
via: "host-bridge",
|
|
2830
|
+
});
|
|
2831
|
+
} finally {
|
|
2832
|
+
client.dispose();
|
|
2833
|
+
}
|
|
2834
|
+
},
|
|
2835
|
+
);
|
|
2836
|
+
|
|
2837
|
+
test.each(["macos", "windows"] as const)(
|
|
2838
|
+
"%s web renderer keeps the existing Playwright fallback without a bridge",
|
|
2839
|
+
async (clientOs) => {
|
|
2840
|
+
const client = getCdpClient(
|
|
2841
|
+
makeContext({
|
|
2842
|
+
conversationId: "conv-native",
|
|
2843
|
+
transportInterface: "web",
|
|
2844
|
+
clientOs,
|
|
2845
|
+
}),
|
|
2846
|
+
);
|
|
2847
|
+
try {
|
|
2848
|
+
expect(await client.send("Page.getFrameTree", {})).toMatchObject({
|
|
2849
|
+
via: "local",
|
|
2850
|
+
});
|
|
2851
|
+
} finally {
|
|
2852
|
+
client.dispose();
|
|
2853
|
+
}
|
|
2854
|
+
},
|
|
2855
|
+
);
|
|
2856
|
+
|
|
2857
|
+
test("explicit Playwright choice overrides the native browser default", async () => {
|
|
2858
|
+
mockSingletonProxy = makeMacosBridgeOnlyProxy();
|
|
2859
|
+
const client = getCdpClient(
|
|
2860
|
+
makeContext({
|
|
2861
|
+
conversationId: "conv-native",
|
|
2862
|
+
transportInterface: "web",
|
|
2863
|
+
clientOs: "windows",
|
|
2864
|
+
}),
|
|
2865
|
+
{ mode: "local" },
|
|
2866
|
+
);
|
|
2867
|
+
try {
|
|
2868
|
+
expect(await client.send("Page.getFrameTree", {})).toMatchObject({
|
|
2869
|
+
via: "local",
|
|
2870
|
+
});
|
|
2871
|
+
} finally {
|
|
2872
|
+
client.dispose();
|
|
2873
|
+
}
|
|
2874
|
+
});
|
|
2875
|
+
});
|
|
@@ -417,6 +417,7 @@ describe("connectCdpWsTransport", () => {
|
|
|
417
417
|
}
|
|
418
418
|
expect(caught).toBeInstanceOf(CdpWsTransportError);
|
|
419
419
|
expect((caught as CdpWsTransportError).code).toBe("closed");
|
|
420
|
+
expect(transport.closed).toBe(true);
|
|
420
421
|
|
|
421
422
|
// After close, subsequent sends also reject with closed.
|
|
422
423
|
let caught2: unknown;
|
|
@@ -99,6 +99,8 @@ export interface CdpTransportEvent {
|
|
|
99
99
|
* CDP error mapping to the shared `CdpError` taxonomy.
|
|
100
100
|
*/
|
|
101
101
|
export interface CdpWsTransport {
|
|
102
|
+
readonly closed?: boolean;
|
|
103
|
+
|
|
102
104
|
/**
|
|
103
105
|
* Send a CDP method call over the socket and await its response.
|
|
104
106
|
*
|
|
@@ -464,6 +466,9 @@ function createTransport(ws: WsLike): CdpWsTransport {
|
|
|
464
466
|
ws.addEventListener("error", handleError);
|
|
465
467
|
|
|
466
468
|
const transport: CdpWsTransport = {
|
|
469
|
+
get closed() {
|
|
470
|
+
return disposed || closed;
|
|
471
|
+
},
|
|
467
472
|
send<T = unknown>(
|
|
468
473
|
method: string,
|
|
469
474
|
params?: Record<string, unknown>,
|
|
@@ -352,7 +352,7 @@ export const computerUseOpenAppTool = {
|
|
|
352
352
|
export const computerUseRunAppleScriptTool = {
|
|
353
353
|
name: "computer_use_run_applescript",
|
|
354
354
|
description:
|
|
355
|
-
"Run an AppleScript
|
|
355
|
+
"Run an AppleScript on the Mac. Try the target app's own scripting dictionary first, asking it for the state you want rather than for the clicks that would produce it; fall back to System Events menu clicking for apps with no dictionary entry for what you need. Prefer this over click and type when the app supports it; it does not move the cursor. The result is the script's return value, and the accessibility tree may not change even when the script worked. Never use 'do shell script' inside AppleScript (blocked for security).",
|
|
356
356
|
category: "computer-use",
|
|
357
357
|
defaultRiskLevel: RiskLevel.Low,
|
|
358
358
|
executionTarget: "host",
|
|
@@ -367,8 +367,7 @@ export const computerUseRunAppleScriptTool = {
|
|
|
367
367
|
},
|
|
368
368
|
reasoning: {
|
|
369
369
|
type: "string",
|
|
370
|
-
description:
|
|
371
|
-
"Explanation of what this script does and why AppleScript is better than UI interaction for this step",
|
|
370
|
+
description: "What this script does",
|
|
372
371
|
},
|
|
373
372
|
target_client_id: {
|
|
374
373
|
type: "string",
|
|
@@ -21,7 +21,18 @@ export type AbortReasonKind =
|
|
|
21
21
|
/** A signal-file cancel was written by an out-of-process caller (CLI, hook). */
|
|
22
22
|
| "signal_cancel"
|
|
23
23
|
/** Voice session bridge aborted the conversation (turn supersession, call end). */
|
|
24
|
-
| "voice_session_aborted"
|
|
24
|
+
| "voice_session_aborted"
|
|
25
|
+
/**
|
|
26
|
+
* One spoken progress update ran out of its generation budget.
|
|
27
|
+
*
|
|
28
|
+
* Distinct from {@link "voice_session_aborted"} because nothing was aborted
|
|
29
|
+
* but the update itself: the call is still up, the turn is still running,
|
|
30
|
+
* and the only consequence is that this beat goes unspoken. Sharing the
|
|
31
|
+
* session-abort kind made a narrator that had gone fail-open read in the
|
|
32
|
+
* logs as a session dying over and over, which is the opposite of what the
|
|
33
|
+
* line means.
|
|
34
|
+
*/
|
|
35
|
+
| "voice_progress_narration_timeout";
|
|
25
36
|
|
|
26
37
|
const ABORT_REASON_TAG = "__vellumAbortReason" as const;
|
|
27
38
|
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the manifest that identifies an installed plugin.
|
|
3
|
+
*
|
|
4
|
+
* `package.json` is the primary manifest for Vellum plugins. A root
|
|
5
|
+
* `plugin.json` is selected only when `package.json` is absent, so a legacy
|
|
6
|
+
* plugin can carry another ecosystem's `plugin.json` without being
|
|
7
|
+
* reclassified.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
|
|
13
|
+
import { z } from "zod";
|
|
14
|
+
|
|
15
|
+
export const LEGACY_PLUGIN_MANIFEST = "package.json";
|
|
16
|
+
export const STANDARD_PLUGIN_MANIFEST = "plugin.json";
|
|
17
|
+
export const AGENT_PLUGINS_MANIFEST_SCHEMA_URL =
|
|
18
|
+
"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json";
|
|
19
|
+
|
|
20
|
+
const PluginNameSchema = z
|
|
21
|
+
.string()
|
|
22
|
+
.min(1)
|
|
23
|
+
.max(64)
|
|
24
|
+
.regex(/^[a-z0-9](?!.*(?:--|\.\.))[a-z0-9.-]*[a-z0-9]$|^[a-z0-9]$/);
|
|
25
|
+
|
|
26
|
+
const LegacyPluginManifestSchema = z
|
|
27
|
+
.object({
|
|
28
|
+
name: z.string().min(1),
|
|
29
|
+
version: z.string().optional(),
|
|
30
|
+
description: z.string().min(1).optional(),
|
|
31
|
+
homepage: z.string().optional(),
|
|
32
|
+
license: z.unknown().optional(),
|
|
33
|
+
peerDependencies: z.record(z.string(), z.string()).optional(),
|
|
34
|
+
credentialKeyPatterns: z.unknown().optional(),
|
|
35
|
+
displayName: z.string().min(1).optional(),
|
|
36
|
+
icon: z.string().min(1).optional(),
|
|
37
|
+
})
|
|
38
|
+
.passthrough();
|
|
39
|
+
|
|
40
|
+
const StandardPluginManifestSchema = z
|
|
41
|
+
.object({
|
|
42
|
+
$schema: z.literal(AGENT_PLUGINS_MANIFEST_SCHEMA_URL),
|
|
43
|
+
name: PluginNameSchema,
|
|
44
|
+
version: z.string().optional(),
|
|
45
|
+
description: z.string().optional(),
|
|
46
|
+
author: z
|
|
47
|
+
.object({
|
|
48
|
+
name: z.string().optional(),
|
|
49
|
+
email: z.string().optional(),
|
|
50
|
+
url: z.string().optional(),
|
|
51
|
+
})
|
|
52
|
+
.strict()
|
|
53
|
+
.optional(),
|
|
54
|
+
homepage: z.string().optional(),
|
|
55
|
+
repository: z.string().optional(),
|
|
56
|
+
license: z.string().optional(),
|
|
57
|
+
keywords: z.array(z.string()).optional(),
|
|
58
|
+
// The core specification says a non-object extensions value is reported
|
|
59
|
+
// and ignored rather than rejecting the plugin. This host does not consume
|
|
60
|
+
// a standard extension namespace here, so keep it opaque.
|
|
61
|
+
extensions: z.unknown().optional(),
|
|
62
|
+
})
|
|
63
|
+
// Unknown core fields are reported-and-ignored by the specification. The
|
|
64
|
+
// loader does not assign them semantics, so passthrough preserves that
|
|
65
|
+
// non-fatal behavior without copying them into runtime metadata.
|
|
66
|
+
.passthrough();
|
|
67
|
+
|
|
68
|
+
export type PluginManifestSource =
|
|
69
|
+
| typeof LEGACY_PLUGIN_MANIFEST
|
|
70
|
+
| typeof STANDARD_PLUGIN_MANIFEST;
|
|
71
|
+
|
|
72
|
+
export interface ResolvedPluginManifest {
|
|
73
|
+
readonly source: PluginManifestSource;
|
|
74
|
+
readonly name: string;
|
|
75
|
+
readonly version?: string;
|
|
76
|
+
readonly description?: string;
|
|
77
|
+
readonly homepage?: string;
|
|
78
|
+
readonly license?: unknown;
|
|
79
|
+
/** Parsed selected manifest, retained for Vellum-specific legacy fields. */
|
|
80
|
+
readonly raw: Record<string, unknown>;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export type PluginManifestInstallAction = "keep" | "synthesize-legacy";
|
|
84
|
+
|
|
85
|
+
export class PluginManifestError extends Error {
|
|
86
|
+
constructor(
|
|
87
|
+
readonly pluginDir: string,
|
|
88
|
+
readonly source: PluginManifestSource | null,
|
|
89
|
+
message: string,
|
|
90
|
+
readonly cause?: unknown,
|
|
91
|
+
) {
|
|
92
|
+
super(message);
|
|
93
|
+
this.name = "PluginManifestError";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** True when the directory carries either supported root manifest. */
|
|
98
|
+
export function hasPluginManifest(pluginDir: string): boolean {
|
|
99
|
+
return (
|
|
100
|
+
existsSync(join(pluginDir, LEGACY_PLUGIN_MANIFEST)) ||
|
|
101
|
+
existsSync(join(pluginDir, STANDARD_PLUGIN_MANIFEST))
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Decide whether an installed artifact needs the compatibility `package.json`.
|
|
107
|
+
*
|
|
108
|
+
* A `package.json` file is always preserved. A root `plugin.json`
|
|
109
|
+
* claims the Agent Plugins format only when it names the supported schema. A
|
|
110
|
+
* claimed standard manifest is validated before install; other `plugin.json`
|
|
111
|
+
* files remain foreign metadata beside the synthesized compatibility manifest.
|
|
112
|
+
*/
|
|
113
|
+
export function getPluginManifestInstallAction(
|
|
114
|
+
pluginDir: string,
|
|
115
|
+
): PluginManifestInstallAction {
|
|
116
|
+
if (existsSync(join(pluginDir, LEGACY_PLUGIN_MANIFEST))) {
|
|
117
|
+
return "keep";
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const standardPath = join(pluginDir, STANDARD_PLUGIN_MANIFEST);
|
|
121
|
+
if (!existsSync(standardPath)) {
|
|
122
|
+
return "synthesize-legacy";
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let json: unknown;
|
|
126
|
+
try {
|
|
127
|
+
json = JSON.parse(readFileSync(standardPath, "utf8"));
|
|
128
|
+
} catch {
|
|
129
|
+
return "synthesize-legacy";
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (
|
|
133
|
+
typeof json !== "object" ||
|
|
134
|
+
json === null ||
|
|
135
|
+
Array.isArray(json) ||
|
|
136
|
+
(json as Record<string, unknown>).$schema !==
|
|
137
|
+
AGENT_PLUGINS_MANIFEST_SCHEMA_URL
|
|
138
|
+
) {
|
|
139
|
+
return "synthesize-legacy";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
readPluginManifest(pluginDir);
|
|
143
|
+
return "keep";
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Read and validate the selected root manifest.
|
|
148
|
+
*
|
|
149
|
+
* Selection is deterministic: `package.json` wins when present, including
|
|
150
|
+
* when it is malformed. The loader never falls through
|
|
151
|
+
* from a selected malformed legacy manifest to `plugin.json`.
|
|
152
|
+
*/
|
|
153
|
+
export function readPluginManifest(pluginDir: string): ResolvedPluginManifest {
|
|
154
|
+
const source = existsSync(join(pluginDir, LEGACY_PLUGIN_MANIFEST))
|
|
155
|
+
? LEGACY_PLUGIN_MANIFEST
|
|
156
|
+
: existsSync(join(pluginDir, STANDARD_PLUGIN_MANIFEST))
|
|
157
|
+
? STANDARD_PLUGIN_MANIFEST
|
|
158
|
+
: null;
|
|
159
|
+
if (source === null) {
|
|
160
|
+
throw new PluginManifestError(
|
|
161
|
+
pluginDir,
|
|
162
|
+
null,
|
|
163
|
+
`plugin at ${pluginDir} is missing package.json and plugin.json`,
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const path = join(pluginDir, source);
|
|
168
|
+
let json: unknown;
|
|
169
|
+
try {
|
|
170
|
+
json = JSON.parse(readFileSync(path, "utf8"));
|
|
171
|
+
} catch (cause) {
|
|
172
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
173
|
+
throw new PluginManifestError(
|
|
174
|
+
pluginDir,
|
|
175
|
+
source,
|
|
176
|
+
`${source} at ${pluginDir} could not be read or parsed: ${reason}`,
|
|
177
|
+
cause,
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const parsed =
|
|
182
|
+
source === LEGACY_PLUGIN_MANIFEST
|
|
183
|
+
? LegacyPluginManifestSchema.safeParse(json)
|
|
184
|
+
: StandardPluginManifestSchema.safeParse(json);
|
|
185
|
+
if (!parsed.success) {
|
|
186
|
+
throw new PluginManifestError(
|
|
187
|
+
pluginDir,
|
|
188
|
+
source,
|
|
189
|
+
`${source} at ${pluginDir} failed schema validation: ${parsed.error.message}`,
|
|
190
|
+
parsed.error,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
source,
|
|
196
|
+
name: parsed.data.name,
|
|
197
|
+
version: parsed.data.version,
|
|
198
|
+
description: parsed.data.description,
|
|
199
|
+
homepage: parsed.data.homepage,
|
|
200
|
+
license: parsed.data.license,
|
|
201
|
+
raw: parsed.data,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { describe, expect, test } from "bun:test";
|
|
2
|
-
|
|
3
|
-
import { setOverridesForTesting } from "../__tests__/feature-flag-test-helpers.js";
|
|
4
|
-
import type { AssistantConfig } from "../config/schema.js";
|
|
5
|
-
import { isAssistantDesktopEnabled } from "./desktop-feature.js";
|
|
6
|
-
|
|
7
|
-
describe("isAssistantDesktopEnabled", () => {
|
|
8
|
-
const config = {} as AssistantConfig;
|
|
9
|
-
|
|
10
|
-
test("needs both the flag and a containerized runtime", () => {
|
|
11
|
-
setOverridesForTesting({ "assistant-desktop": true });
|
|
12
|
-
expect(isAssistantDesktopEnabled(config, true)).toBe(true);
|
|
13
|
-
expect(isAssistantDesktopEnabled(config, false)).toBe(false);
|
|
14
|
-
|
|
15
|
-
setOverridesForTesting({ "assistant-desktop": false });
|
|
16
|
-
expect(isAssistantDesktopEnabled(config, true)).toBe(false);
|
|
17
|
-
});
|
|
18
|
-
});
|