@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/desktop-browser-cli.md +33 -0
- package/openapi.yaml +17 -5
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +279 -0
- package/src/__tests__/config-schema.test.ts +1 -1
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/host-cu-proxy.test.ts +116 -18
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/mcp-auth-routes.test.ts +99 -0
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-health-check.test.ts +32 -32
- package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/skills.test.ts +10 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/acp/session-snapshot.ts +260 -0
- package/src/apps/app-store.ts +6 -5
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/calls/__tests__/progress-narration.test.ts +23 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
- package/src/calls/progress-narration.ts +15 -2
- package/src/calls/voice-control-protocol.ts +34 -4
- package/src/calls/voice-session-bridge.ts +23 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/plugins.ts +16 -5
- package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +13 -0
- package/src/cli/lib/install-from-github.ts +17 -18
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-details.ts +21 -8
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/bundled-skills/acp/SKILL.md +1 -1
- package/src/config/bundled-skills/acp/TOOLS.json +2 -2
- package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
- package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
- package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
- package/src/config/feature-flag-registry.json +2 -2
- package/src/config/schemas/__tests__/voice.test.ts +2 -2
- package/src/config/schemas/mcp.ts +9 -7
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/voice.ts +2 -2
- package/src/config/skills.ts +13 -21
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-runtime-assembly.ts +1 -1
- package/src/daemon/conversation-tool-setup.ts +31 -29
- package/src/daemon/host-cu-proxy.ts +51 -26
- package/src/daemon/host-proxy-preactivation.ts +2 -0
- package/src/daemon/mcp-reload-service.ts +7 -0
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/providers-setup.ts +4 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-automation-lease.test.ts +208 -0
- package/src/desktop/desktop-automation-lease.ts +270 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +58 -8
- package/src/desktop/desktop-dependencies.test.ts +29 -0
- package/src/desktop/desktop-dependencies.ts +26 -5
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.ts +14 -1
- package/src/desktop/desktop-session-manager.test.ts +86 -3
- package/src/desktop/desktop-session-manager.ts +144 -24
- package/src/desktop/desktop-stream-bridge.ts +1 -1
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/i18n/__tests__/i18n.test.ts +9 -5
- package/src/i18n/messages.ts +13 -4
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/protocol.test.ts +11 -3
- package/src/live-voice/__tests__/session-controls.test.ts +34 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-session.ts +3 -0
- package/src/live-voice/protocol.ts +10 -1
- package/src/live-voice/session-controls.ts +28 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +11 -0
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +42 -15
- package/src/mcp/manager.ts +61 -8
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/notifications/__tests__/decision-engine.test.ts +175 -0
- package/src/notifications/decision-engine.ts +43 -8
- package/src/oauth/seed-providers.ts +11 -0
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +9 -15
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
- package/src/providers/inference/endpoint-probe.ts +27 -12
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +167 -0
- package/src/providers/opencode/client.ts +73 -4
- package/src/runtime/AGENTS.md +4 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
- package/src/runtime/http-server.ts +6 -6
- package/src/runtime/routes/__tests__/acp-routes.test.ts +19 -0
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
- package/src/runtime/routes/acp-routes.ts +17 -161
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
- package/src/runtime/routes/desktop-setup-routes.ts +7 -5
- package/src/runtime/routes/mcp-auth-routes.ts +106 -94
- package/src/runtime/routes/plugins-routes.ts +14 -1
- package/src/runtime/sync/resource-sync-events.ts +4 -0
- package/src/tools/acp/status.test.ts +276 -21
- package/src/tools/acp/status.ts +98 -32
- package/src/tools/browser/browser-execution.ts +44 -13
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/computer-use/definitions.ts +2 -3
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/plugin-manifest.ts +203 -0
- package/src/desktop/desktop-feature.test.ts +0 -18
- package/src/desktop/desktop-feature.ts +0 -19
|
@@ -2146,6 +2146,68 @@ describe("front-door leg tool suppression", () => {
|
|
|
2146
2146
|
});
|
|
2147
2147
|
});
|
|
2148
2148
|
|
|
2149
|
+
describe("desktop skill preactivation", () => {
|
|
2150
|
+
async function preactivatedFor(turn: Record<string, unknown>): Promise<{
|
|
2151
|
+
skillIds: string[];
|
|
2152
|
+
proxyInterfaces: unknown[];
|
|
2153
|
+
skillIdsDuringLoop: string[];
|
|
2154
|
+
}> {
|
|
2155
|
+
const skillIds: string[] = [];
|
|
2156
|
+
const proxyInterfaces: unknown[] = [];
|
|
2157
|
+
let skillIdsDuringLoop: string[] = [];
|
|
2158
|
+
const fake = makeFakeConversation({
|
|
2159
|
+
processing: false,
|
|
2160
|
+
runAgentLoop: async () => {
|
|
2161
|
+
skillIdsDuringLoop = [...skillIds];
|
|
2162
|
+
},
|
|
2163
|
+
});
|
|
2164
|
+
Object.assign(fake.conversation, {
|
|
2165
|
+
addPreactivatedSkillId: (id: string) => {
|
|
2166
|
+
skillIds.push(id);
|
|
2167
|
+
},
|
|
2168
|
+
ensureHostProxiesForTurn: (sourceInterface: unknown) => {
|
|
2169
|
+
proxyInterfaces.push(sourceInterface);
|
|
2170
|
+
},
|
|
2171
|
+
});
|
|
2172
|
+
fakeConversation = fake.conversation;
|
|
2173
|
+
|
|
2174
|
+
await startVoiceTurn({
|
|
2175
|
+
...makeTurnOptions(undefined, "conv-desktop-skills"),
|
|
2176
|
+
userMessageInterface: "macos",
|
|
2177
|
+
...turn,
|
|
2178
|
+
});
|
|
2179
|
+
await flushMicrotasks();
|
|
2180
|
+
return { skillIds, proxyInterfaces, skillIdsDuringLoop };
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
|
|
2184
|
+
const result = await preactivatedFor({
|
|
2185
|
+
routingLeg: "escalated",
|
|
2186
|
+
macosDesktopSession: true,
|
|
2187
|
+
});
|
|
2188
|
+
|
|
2189
|
+
expect(result.skillIdsDuringLoop).toContain("computer-use");
|
|
2190
|
+
expect(result.proxyInterfaces).toEqual(["macos"]);
|
|
2191
|
+
});
|
|
2192
|
+
|
|
2193
|
+
test("a session from any other client leaves the desktop skills to be loaded", async () => {
|
|
2194
|
+
const result = await preactivatedFor({ routingLeg: "escalated" });
|
|
2195
|
+
|
|
2196
|
+
expect(result.skillIds).toEqual([]);
|
|
2197
|
+
expect(result.proxyInterfaces).toEqual([]);
|
|
2198
|
+
});
|
|
2199
|
+
|
|
2200
|
+
test("the toolless front-door leg is never preactivated", async () => {
|
|
2201
|
+
const result = await preactivatedFor({
|
|
2202
|
+
routingLeg: "front-door",
|
|
2203
|
+
macosDesktopSession: true,
|
|
2204
|
+
});
|
|
2205
|
+
|
|
2206
|
+
expect(result.skillIds).toEqual([]);
|
|
2207
|
+
expect(result.proxyInterfaces).toEqual([]);
|
|
2208
|
+
});
|
|
2209
|
+
});
|
|
2210
|
+
|
|
2149
2211
|
describe("cutFrontDoorContentAtVerdict", () => {
|
|
2150
2212
|
test("null when the content carries no verdict token (committed answer)", () => {
|
|
2151
2213
|
expect(
|
|
@@ -16,7 +16,7 @@ import type {
|
|
|
16
16
|
ProviderResponse,
|
|
17
17
|
ToolDefinition,
|
|
18
18
|
} from "../providers/types.js";
|
|
19
|
-
import { createAbortReason } from "../util/abort-reasons.js";
|
|
19
|
+
import { createAbortReason, isAbortReason } from "../util/abort-reasons.js";
|
|
20
20
|
import { getLogger } from "../util/logger.js";
|
|
21
21
|
|
|
22
22
|
const log = getLogger("voice-progress-narration");
|
|
@@ -164,7 +164,10 @@ async function requestBoundedResponse(args: {
|
|
|
164
164
|
const timeoutTimer = setTimeout(
|
|
165
165
|
() =>
|
|
166
166
|
timeoutController.abort(
|
|
167
|
-
createAbortReason(
|
|
167
|
+
createAbortReason(
|
|
168
|
+
"voice_progress_narration_timeout",
|
|
169
|
+
"voice-progress-narration",
|
|
170
|
+
),
|
|
168
171
|
),
|
|
169
172
|
args.timeoutMs,
|
|
170
173
|
);
|
|
@@ -260,9 +263,19 @@ export function createVoiceProgressNarrator(options: {
|
|
|
260
263
|
}
|
|
261
264
|
return trimmed;
|
|
262
265
|
} catch (error) {
|
|
266
|
+
// Why the beat went unspoken, as one greppable field. A narrator whose
|
|
267
|
+
// budget is below the model's real roundtrip drops every update at the
|
|
268
|
+
// deadline and reads in the logs as a run of unrelated failures; naming
|
|
269
|
+
// the outcome is what makes that run legible as "the feature is off".
|
|
270
|
+
const outcome = isAbortReason(error)
|
|
271
|
+
? error.kind === "voice_progress_narration_timeout"
|
|
272
|
+
? "timed_out"
|
|
273
|
+
: "session_ended"
|
|
274
|
+
: "error";
|
|
263
275
|
log.info(
|
|
264
276
|
{
|
|
265
277
|
error,
|
|
278
|
+
outcome,
|
|
266
279
|
providerResolveMs,
|
|
267
280
|
totalMs: Math.round(performance.now() - startedAt),
|
|
268
281
|
timeoutMs: options.config.generationTimeoutMs,
|
|
@@ -60,6 +60,18 @@ export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
|
|
|
60
60
|
export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
|
|
61
61
|
const UPDATES_MARKER_PREFIX = "[UPDATES:";
|
|
62
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Look session controls for live voice: turn on a screen share
|
|
65
|
+
* ({@link LOOK_SCREEN_MARKER}) or the camera ({@link LOOK_CAMERA_MARKER}) so
|
|
66
|
+
* the call can see what the user means by "take a look". Carried out by the
|
|
67
|
+
* client, like mute and end.
|
|
68
|
+
*/
|
|
69
|
+
export const LOOK_SCREEN_MARKER = "[LOOK:SCREEN]";
|
|
70
|
+
export const LOOK_CAMERA_MARKER = "[LOOK:CAMERA]";
|
|
71
|
+
/** Stop showing the call the screen and the camera, whichever is on. */
|
|
72
|
+
export const LOOK_STOP_MARKER = "[LOOK:STOP]";
|
|
73
|
+
const LOOK_MARKER_PREFIX = "[LOOK:";
|
|
74
|
+
|
|
63
75
|
// ---------------------------------------------------------------------------
|
|
64
76
|
// Regexes
|
|
65
77
|
// ---------------------------------------------------------------------------
|
|
@@ -82,6 +94,7 @@ const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
|
|
|
82
94
|
const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
|
|
83
95
|
const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
|
|
84
96
|
const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
|
|
97
|
+
const LOOK_MARKER_REGEX = /\[LOOK:\s*[^\]]*\]/g;
|
|
85
98
|
const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
|
|
86
99
|
const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
|
|
87
100
|
|
|
@@ -209,6 +222,7 @@ export function stripInternalSpeechMarkers(text: string): string {
|
|
|
209
222
|
.replace(MINIMIZE_ROOM_MARKER_REGEX, "")
|
|
210
223
|
.replace(MUTE_MARKER_REGEX, "")
|
|
211
224
|
.replace(UPDATES_MARKER_REGEX, "")
|
|
225
|
+
.replace(LOOK_MARKER_REGEX, "")
|
|
212
226
|
.replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
|
|
213
227
|
.replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
|
|
214
228
|
return result;
|
|
@@ -237,6 +251,7 @@ const CONTROL_MARKER_STRINGS = [
|
|
|
237
251
|
MUTE_MARKER,
|
|
238
252
|
MUTE_MARKER_PREFIX,
|
|
239
253
|
UPDATES_MARKER_PREFIX,
|
|
254
|
+
LOOK_MARKER_PREFIX,
|
|
240
255
|
"[GUARDIAN_TIMEOUT]",
|
|
241
256
|
"[GUARDIAN_UNAVAILABLE]",
|
|
242
257
|
];
|
|
@@ -252,6 +267,7 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
|
|
|
252
267
|
"[USER_INSTRUCTION:",
|
|
253
268
|
MUTE_MARKER_PREFIX,
|
|
254
269
|
UPDATES_MARKER_PREFIX,
|
|
270
|
+
LOOK_MARKER_PREFIX,
|
|
255
271
|
];
|
|
256
272
|
|
|
257
273
|
const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
|
|
@@ -339,15 +355,19 @@ export function createControlMarkerHoldback(
|
|
|
339
355
|
/**
|
|
340
356
|
* A session control a live-voice reply asked for with a terminal marker:
|
|
341
357
|
* `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
|
|
342
|
-
* timed form, `updates` from the progress-cadence markers
|
|
358
|
+
* timed form, `updates` from the progress-cadence markers, `look_screen` and
|
|
359
|
+
* `look_camera` and `look_stop` from the look markers.
|
|
343
360
|
*/
|
|
344
361
|
export type SessionControlRequest =
|
|
345
362
|
| { readonly action: "end" }
|
|
346
363
|
| { readonly action: "mute"; readonly durationMs?: number }
|
|
347
|
-
| { readonly action: "updates"; readonly cadence: "fewer" | "normal" }
|
|
364
|
+
| { readonly action: "updates"; readonly cadence: "fewer" | "normal" }
|
|
365
|
+
| { readonly action: "look_screen" }
|
|
366
|
+
| { readonly action: "look_camera" }
|
|
367
|
+
| { readonly action: "look_stop" };
|
|
348
368
|
|
|
349
369
|
const TERMINAL_SESSION_CONTROL_REGEX =
|
|
350
|
-
/(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
|
|
370
|
+
/(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[LOOK:(SCREEN|CAMERA|STOP)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
|
|
351
371
|
|
|
352
372
|
/**
|
|
353
373
|
* The session control a reply ends with, or null.
|
|
@@ -378,7 +398,17 @@ export function parseTerminalSessionControl(
|
|
|
378
398
|
cadence: match[2] === "FEWER" ? "fewer" : "normal",
|
|
379
399
|
};
|
|
380
400
|
}
|
|
381
|
-
|
|
401
|
+
if (match[3] !== undefined) {
|
|
402
|
+
return {
|
|
403
|
+
action:
|
|
404
|
+
match[3] === "SCREEN"
|
|
405
|
+
? "look_screen"
|
|
406
|
+
: match[3] === "CAMERA"
|
|
407
|
+
? "look_camera"
|
|
408
|
+
: "look_stop",
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
const seconds = match[4] === undefined ? NaN : Number(match[4].trim());
|
|
382
412
|
if (
|
|
383
413
|
Number.isFinite(seconds) &&
|
|
384
414
|
seconds > 0 &&
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
|
|
32
32
|
import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
|
|
33
33
|
import { getOrCreateConversation } from "../daemon/conversation-store.js";
|
|
34
|
+
import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
|
|
34
35
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
35
36
|
import {
|
|
36
37
|
newestPersistedSightFrame,
|
|
@@ -464,6 +465,14 @@ export interface VoiceTurnOptions {
|
|
|
464
465
|
* path has always used.
|
|
465
466
|
*/
|
|
466
467
|
actorFallbackSuppressed?: boolean;
|
|
468
|
+
/**
|
|
469
|
+
* Whether the session was opened from the macOS desktop client. A leg that
|
|
470
|
+
* can run tools then starts with the desktop skills (computer use, app
|
|
471
|
+
* control, screen annotation) already active, as a typed turn from that
|
|
472
|
+
* client does. The turn interface cannot answer this: every live-voice
|
|
473
|
+
* session reports `macos` for its channel capabilities, iOS included.
|
|
474
|
+
*/
|
|
475
|
+
macosDesktopSession?: boolean;
|
|
467
476
|
/** Whether this is an inbound call (no outbound task). */
|
|
468
477
|
isInbound: boolean;
|
|
469
478
|
/** The outbound call task, if any. */
|
|
@@ -2016,6 +2025,20 @@ export async function startVoiceTurn(
|
|
|
2016
2025
|
(needsImagePin
|
|
2017
2026
|
? VOICE_IMAGE_PROFILE
|
|
2018
2027
|
: (conversationProfile?.profile ?? null));
|
|
2028
|
+
if (opts.macosDesktopSession === true && !frontDoorToolsSuppressed) {
|
|
2029
|
+
const sourceInterface = turnInterfaceContext.userMessageInterface;
|
|
2030
|
+
const sourceActorPrincipalId =
|
|
2031
|
+
voiceTurnValues.actorPrincipalId ?? undefined;
|
|
2032
|
+
conversation.ensureHostProxiesForTurn(
|
|
2033
|
+
sourceInterface,
|
|
2034
|
+
sourceActorPrincipalId,
|
|
2035
|
+
);
|
|
2036
|
+
preactivateHostProxySkills(
|
|
2037
|
+
conversation,
|
|
2038
|
+
sourceInterface,
|
|
2039
|
+
sourceActorPrincipalId,
|
|
2040
|
+
);
|
|
2041
|
+
}
|
|
2019
2042
|
await conversation.runAgentLoop(persistedContent, messageId, {
|
|
2020
2043
|
onEvent: (msg: AssistantEvent) => {
|
|
2021
2044
|
if (msg.type === "assistant_turn_start") {
|
|
@@ -67,9 +67,8 @@ mock.module("../../../util/logger.js", () => ({
|
|
|
67
67
|
// Import module under test (after mocks)
|
|
68
68
|
// ---------------------------------------------------------------------------
|
|
69
69
|
|
|
70
|
-
const { formatBrowserStatusLines, registerBrowserCommand } =
|
|
71
|
-
"../browser.js"
|
|
72
|
-
);
|
|
70
|
+
const { formatBrowserStatusLines, registerBrowserCommand } =
|
|
71
|
+
await import("../browser.js");
|
|
73
72
|
|
|
74
73
|
// ---------------------------------------------------------------------------
|
|
75
74
|
// Test helper
|
|
@@ -641,3 +640,85 @@ describe("error exit codes", () => {
|
|
|
641
640
|
expect(exitCode).toBe(0);
|
|
642
641
|
});
|
|
643
642
|
});
|
|
643
|
+
|
|
644
|
+
for (const desktopFlag of ["--virtual-desktop", "--desktop"]) {
|
|
645
|
+
describe(`virtual desktop routing via ${desktopFlag}`, () => {
|
|
646
|
+
test("routes semantic actions through IPC with conversation and element identity", async () => {
|
|
647
|
+
process.env.__CONVERSATION_ID = "conv-desktop";
|
|
648
|
+
const result = await runCommand([
|
|
649
|
+
"browser",
|
|
650
|
+
desktopFlag,
|
|
651
|
+
"--json",
|
|
652
|
+
"click",
|
|
653
|
+
"--element-id",
|
|
654
|
+
"e7",
|
|
655
|
+
]);
|
|
656
|
+
expect(result.exitCode).toBe(0);
|
|
657
|
+
expect(lastIpcCall?.method).toBe("browser_execute");
|
|
658
|
+
expect(lastBody()).toMatchObject({
|
|
659
|
+
desktop: true,
|
|
660
|
+
conversationId: "conv-desktop",
|
|
661
|
+
operation: "click",
|
|
662
|
+
input: { element_id: "e7" },
|
|
663
|
+
});
|
|
664
|
+
expect(
|
|
665
|
+
(lastBody().input as Record<string, unknown>).desktop,
|
|
666
|
+
).toBeUndefined();
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
test.each([
|
|
670
|
+
{ args: ["list"] },
|
|
671
|
+
{ args: ["select", "--tab-id", "7"] },
|
|
672
|
+
{ args: ["new"] },
|
|
673
|
+
{ args: ["close", "--tab-id", "7"] },
|
|
674
|
+
])("routes desktop tabs $args through IPC", async ({ args }) => {
|
|
675
|
+
mockIpcResult = { ok: true, result: { ok: true, tabs: [] } };
|
|
676
|
+
const result = await runCommand([
|
|
677
|
+
"browser",
|
|
678
|
+
desktopFlag,
|
|
679
|
+
"--json",
|
|
680
|
+
"tabs",
|
|
681
|
+
...args,
|
|
682
|
+
]);
|
|
683
|
+
expect(result.exitCode).toBe(0);
|
|
684
|
+
expect(lastIpcCall?.method).toBe("browser_tabs");
|
|
685
|
+
expect(lastBody()).toMatchObject({ desktop: true, command: args[0] });
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
test.each([
|
|
689
|
+
{ flags: ["--browser-mode", "extension"] },
|
|
690
|
+
{ flags: ["--target-client-id", "client-123"] },
|
|
691
|
+
])("rejects conflicting desktop targets before IPC", async ({ flags }) => {
|
|
692
|
+
const result = await runCommand([
|
|
693
|
+
"browser",
|
|
694
|
+
desktopFlag,
|
|
695
|
+
...flags,
|
|
696
|
+
"tabs",
|
|
697
|
+
"list",
|
|
698
|
+
]);
|
|
699
|
+
expect(result.exitCode).not.toBe(0);
|
|
700
|
+
expect(lastIpcCall).toBeNull();
|
|
701
|
+
});
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
test.each([
|
|
706
|
+
{ args: ["list"] },
|
|
707
|
+
{ args: ["select", "--tab-id", "1"] },
|
|
708
|
+
{ args: ["new", "--url", "https://example.com"] },
|
|
709
|
+
{ args: ["close", "--tab-id", "1"] },
|
|
710
|
+
])(
|
|
711
|
+
"tab command $args forwards an explicit backend override",
|
|
712
|
+
async ({ args }) => {
|
|
713
|
+
mockIpcResult = { ok: true, result: { ok: true, tabs: [] } };
|
|
714
|
+
await runCommand([
|
|
715
|
+
"browser",
|
|
716
|
+
"--browser-mode",
|
|
717
|
+
"extension",
|
|
718
|
+
"tabs",
|
|
719
|
+
...args,
|
|
720
|
+
]);
|
|
721
|
+
expect(lastIpcCall?.method).toBe("browser_tabs");
|
|
722
|
+
expect(lastBody().browserMode).toBe("extension");
|
|
723
|
+
},
|
|
724
|
+
);
|
|
@@ -96,6 +96,15 @@ export const browserHelp: CliCommandHelp = {
|
|
|
96
96
|
name: "browser",
|
|
97
97
|
description: "Control the browser via the running assistant.",
|
|
98
98
|
options: [
|
|
99
|
+
{
|
|
100
|
+
flags: "--desktop",
|
|
101
|
+
description: "Compatibility alias for --virtual-desktop.",
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
flags: "--virtual-desktop",
|
|
105
|
+
description:
|
|
106
|
+
"Use Chrome in the platform-hosted virtual desktop with visible cursor feedback.",
|
|
107
|
+
},
|
|
99
108
|
{
|
|
100
109
|
flags: "--session <id>",
|
|
101
110
|
description: "Session ID to preserve browser state across invocations.",
|
|
@@ -118,6 +127,38 @@ export const browserHelp: CliCommandHelp = {
|
|
|
118
127
|
],
|
|
119
128
|
helpText: `
|
|
120
129
|
Browser operations are executed through the running assistant.
|
|
130
|
+
Virtual desktop is available only for platform-hosted assistants.
|
|
131
|
+
In the web client, automatic selection prefers virtual desktop Chrome when
|
|
132
|
+
its feature is enabled and the conversation has an
|
|
133
|
+
identified guardian. The macOS and Windows apps keep their existing browser
|
|
134
|
+
selection and fallback behavior; use --virtual-desktop to select it there.
|
|
135
|
+
Existing sessions and explicit backend/client choices take priority.
|
|
136
|
+
--browser-mode local (alias playwright) means the assistant's Playwright browser,
|
|
137
|
+
not the user's Chrome. --desktop remains an alias for --virtual-desktop.
|
|
138
|
+
Use --virtual-desktop for Chrome in the Desktop panel.
|
|
139
|
+
Run navigate --url <url> directly to open a page.
|
|
140
|
+
The assistant-desktop feature flag and completed installation are required.
|
|
141
|
+
The first browser command installs missing components, starts Chrome, and completes
|
|
142
|
+
the requested action in one call. Allow up to 600 seconds for this command when
|
|
143
|
+
using bash (timeout_seconds: 600). Opening the Desktop panel also starts
|
|
144
|
+
the same installation and shows progress. No separate setup command is needed.
|
|
145
|
+
Report other availability errors without switching to a personal computer.
|
|
146
|
+
The assistant manages Chrome startup. Do not launch a separate desktop stack,
|
|
147
|
+
install desktop packages, or drive webpages with shell-level xdotool.
|
|
148
|
+
With --virtual-desktop, screenshot captures the page directly from Chrome over CDP
|
|
149
|
+
as a color JPEG. Use --output to save it and file_read to view it. Do not use
|
|
150
|
+
shell-level xwd or custom screenshot conversion scripts.
|
|
151
|
+
Use snapshot for current element IDs and tabs list for current tab IDs.
|
|
152
|
+
Refresh the snapshot after navigation, tab changes or stale-element errors.
|
|
153
|
+
Inspect the result before retrying a failed action; it may have happened.
|
|
154
|
+
A purple page pointer shows CDP mouse movement in the stream. Browser toolbar
|
|
155
|
+
controls, native dialogs and other apps are outside this CLI's page controls.
|
|
156
|
+
Do not combine --virtual-desktop with personal browser targets, other browser modes
|
|
157
|
+
or --use-active-tab. Download waiting is unavailable on this target.
|
|
158
|
+
Users can interact directly with the expanded desktop. If they ask you to pause,
|
|
159
|
+
stop and wait. Take a fresh snapshot before resuming. Closing the viewer does not
|
|
160
|
+
end automation. Run assistant browser --virtual-desktop detach when finished or
|
|
161
|
+
blocked, including before asking a question. Chrome stays running.
|
|
121
162
|
Each subcommand maps to a browser operation and communicates
|
|
122
163
|
with the assistant process.
|
|
123
164
|
|
|
@@ -131,6 +172,8 @@ local. Useful for debugging or when deterministic backend selection
|
|
|
131
172
|
is required.
|
|
132
173
|
|
|
133
174
|
Examples:
|
|
175
|
+
$ assistant browser --virtual-desktop status
|
|
176
|
+
$ assistant browser --virtual-desktop screenshot --output /tmp/desktop-page.jpg
|
|
134
177
|
$ assistant browser navigate --url https://example.com
|
|
135
178
|
$ assistant browser snapshot
|
|
136
179
|
$ assistant browser click --selector "#login"
|
|
@@ -28,6 +28,8 @@ import { registerCommand } from "../lib/register-command.js";
|
|
|
28
28
|
import { log } from "../logger.js";
|
|
29
29
|
import { browserHelp, toKebab } from "./browser.help.js";
|
|
30
30
|
|
|
31
|
+
const BROWSER_REQUEST_TIMEOUT_MS = 10 * 60_000;
|
|
32
|
+
|
|
31
33
|
// ── Naming helpers ───────────────────────────────────────────────────
|
|
32
34
|
|
|
33
35
|
/**
|
|
@@ -116,6 +118,7 @@ function attachOperationAction(
|
|
|
116
118
|
|
|
117
119
|
subcmd.action(async (opts: Record<string, unknown>) => {
|
|
118
120
|
const parentOpts = browser.opts() as {
|
|
121
|
+
desktop?: boolean;
|
|
119
122
|
session?: string;
|
|
120
123
|
json?: boolean;
|
|
121
124
|
browserMode?: string;
|
|
@@ -130,6 +133,8 @@ function attachOperationAction(
|
|
|
130
133
|
// targetClientId) and screenshot ergonomics (output).
|
|
131
134
|
const input: Record<string, unknown> = {};
|
|
132
135
|
const excludeKeys = new Set([
|
|
136
|
+
"desktop",
|
|
137
|
+
"virtualDesktop",
|
|
133
138
|
"session",
|
|
134
139
|
"json",
|
|
135
140
|
"output",
|
|
@@ -163,9 +168,7 @@ function attachOperationAction(
|
|
|
163
168
|
}
|
|
164
169
|
}
|
|
165
170
|
|
|
166
|
-
//
|
|
167
|
-
// challenges, downloads up to 120s, etc.), so use a generous
|
|
168
|
-
// IPC timeout that exceeds any server-side operation timeout.
|
|
171
|
+
// First use can include virtual desktop installation and Chrome startup.
|
|
169
172
|
const ipcResult = await cliIpcCall<BrowserExecuteResult>(
|
|
170
173
|
"browser_execute",
|
|
171
174
|
{
|
|
@@ -173,10 +176,11 @@ function attachOperationAction(
|
|
|
173
176
|
operation: meta.operation,
|
|
174
177
|
input,
|
|
175
178
|
sessionId,
|
|
179
|
+
...(parentOpts.desktop ? { desktop: true } : {}),
|
|
176
180
|
...(conversationId ? { conversationId } : {}),
|
|
177
181
|
},
|
|
178
182
|
},
|
|
179
|
-
{ timeoutMs:
|
|
183
|
+
{ timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
|
|
180
184
|
);
|
|
181
185
|
|
|
182
186
|
if (!ipcResult.ok) {
|
|
@@ -328,6 +332,22 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
328
332
|
description: browserHelp.description,
|
|
329
333
|
build: (browser) => {
|
|
330
334
|
applyCommandHelp(browser, browserHelp);
|
|
335
|
+
browser.hook("preAction", () => {
|
|
336
|
+
if (browser.opts().virtualDesktop) {
|
|
337
|
+
browser.setOptionValue("desktop", true);
|
|
338
|
+
}
|
|
339
|
+
const options = browser.opts();
|
|
340
|
+
if (
|
|
341
|
+
options.desktop &&
|
|
342
|
+
(options.targetClientId ||
|
|
343
|
+
(options.browserMode &&
|
|
344
|
+
!["auto", "cdp-inspect"].includes(options.browserMode)))
|
|
345
|
+
) {
|
|
346
|
+
throw new Error(
|
|
347
|
+
"--virtual-desktop cannot be combined with a personal browser target or another browser mode",
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
});
|
|
331
351
|
|
|
332
352
|
// Attach one action per browser operation
|
|
333
353
|
for (const meta of BROWSER_OPERATION_META) {
|
|
@@ -339,6 +359,8 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
339
359
|
|
|
340
360
|
subcommand(tabs, "list").action(async (opts: { pretty?: boolean }) => {
|
|
341
361
|
const parentOpts = browser.opts() as {
|
|
362
|
+
desktop?: boolean;
|
|
363
|
+
browserMode?: string;
|
|
342
364
|
session?: string;
|
|
343
365
|
json?: boolean;
|
|
344
366
|
targetClientId?: string;
|
|
@@ -364,11 +386,15 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
364
386
|
body: {
|
|
365
387
|
command: "list",
|
|
366
388
|
sessionId,
|
|
389
|
+
...(parentOpts.desktop ? { desktop: true } : {}),
|
|
390
|
+
...(parentOpts.browserMode
|
|
391
|
+
? { browserMode: parentOpts.browserMode }
|
|
392
|
+
: {}),
|
|
367
393
|
...(conversationId ? { conversationId } : {}),
|
|
368
394
|
...(targetClientId ? { targetClientId } : {}),
|
|
369
395
|
},
|
|
370
396
|
},
|
|
371
|
-
{ timeoutMs:
|
|
397
|
+
{ timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
|
|
372
398
|
);
|
|
373
399
|
|
|
374
400
|
if (!ipcResult.ok) {
|
|
@@ -409,6 +435,8 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
409
435
|
|
|
410
436
|
subcommand(tabs, "select").action(async (opts: { tabId: string }) => {
|
|
411
437
|
const parentOpts = browser.opts() as {
|
|
438
|
+
desktop?: boolean;
|
|
439
|
+
browserMode?: string;
|
|
412
440
|
session?: string;
|
|
413
441
|
json?: boolean;
|
|
414
442
|
targetClientId?: string;
|
|
@@ -425,12 +453,16 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
425
453
|
body: {
|
|
426
454
|
command: "select",
|
|
427
455
|
sessionId,
|
|
456
|
+
...(parentOpts.desktop ? { desktop: true } : {}),
|
|
457
|
+
...(parentOpts.browserMode
|
|
458
|
+
? { browserMode: parentOpts.browserMode }
|
|
459
|
+
: {}),
|
|
428
460
|
tabId,
|
|
429
461
|
...(conversationId ? { conversationId } : {}),
|
|
430
462
|
...(targetClientId ? { targetClientId } : {}),
|
|
431
463
|
},
|
|
432
464
|
},
|
|
433
|
-
{ timeoutMs:
|
|
465
|
+
{ timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
|
|
434
466
|
);
|
|
435
467
|
|
|
436
468
|
if (!ipcResult.ok) {
|
|
@@ -456,6 +488,8 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
456
488
|
|
|
457
489
|
subcommand(tabs, "new").action(async (opts: { url?: string }) => {
|
|
458
490
|
const parentOpts = browser.opts() as {
|
|
491
|
+
desktop?: boolean;
|
|
492
|
+
browserMode?: string;
|
|
459
493
|
session?: string;
|
|
460
494
|
json?: boolean;
|
|
461
495
|
targetClientId?: string;
|
|
@@ -475,12 +509,16 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
475
509
|
body: {
|
|
476
510
|
command: "new",
|
|
477
511
|
sessionId,
|
|
512
|
+
...(parentOpts.desktop ? { desktop: true } : {}),
|
|
513
|
+
...(parentOpts.browserMode
|
|
514
|
+
? { browserMode: parentOpts.browserMode }
|
|
515
|
+
: {}),
|
|
478
516
|
...(opts.url ? { url: opts.url } : {}),
|
|
479
517
|
...(conversationId ? { conversationId } : {}),
|
|
480
518
|
...(targetClientId ? { targetClientId } : {}),
|
|
481
519
|
},
|
|
482
520
|
},
|
|
483
|
-
{ timeoutMs:
|
|
521
|
+
{ timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
|
|
484
522
|
);
|
|
485
523
|
|
|
486
524
|
if (!ipcResult.ok) {
|
|
@@ -512,6 +550,8 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
512
550
|
|
|
513
551
|
subcommand(tabs, "close").action(async (opts: { tabId: string }) => {
|
|
514
552
|
const parentOpts = browser.opts() as {
|
|
553
|
+
desktop?: boolean;
|
|
554
|
+
browserMode?: string;
|
|
515
555
|
session?: string;
|
|
516
556
|
json?: boolean;
|
|
517
557
|
targetClientId?: string;
|
|
@@ -532,12 +572,16 @@ export function registerBrowserCommand(program: Command): void {
|
|
|
532
572
|
body: {
|
|
533
573
|
command: "close",
|
|
534
574
|
sessionId,
|
|
575
|
+
...(parentOpts.desktop ? { desktop: true } : {}),
|
|
576
|
+
...(parentOpts.browserMode
|
|
577
|
+
? { browserMode: parentOpts.browserMode }
|
|
578
|
+
: {}),
|
|
535
579
|
tabId,
|
|
536
580
|
...(conversationId ? { conversationId } : {}),
|
|
537
581
|
...(targetClientId ? { targetClientId } : {}),
|
|
538
582
|
},
|
|
539
583
|
},
|
|
540
|
-
{ timeoutMs:
|
|
584
|
+
{ timeoutMs: BROWSER_REQUEST_TIMEOUT_MS, cancelOnDisconnect: true },
|
|
541
585
|
);
|
|
542
586
|
|
|
543
587
|
if (!ipcResult.ok) {
|
|
@@ -53,6 +53,7 @@ import {
|
|
|
53
53
|
PluginAlreadyInStateException,
|
|
54
54
|
PluginDirectoryNotFoundError,
|
|
55
55
|
} from "../lib/toggle-plugin.js";
|
|
56
|
+
import type { PluginUninstallWarningKey } from "../lib/uninstall-plugin.js";
|
|
56
57
|
import type { PluginUpgradeResult } from "../lib/upgrade-plugin.js";
|
|
57
58
|
import { getCliLogger } from "../logger.js";
|
|
58
59
|
import { PLUGINS_SEARCH_INSTALL_HINT, pluginsHelp } from "./plugins.help.js";
|
|
@@ -666,11 +667,16 @@ export function registerPluginsCommand(program: Command): void {
|
|
|
666
667
|
// when the daemon is unreachable (a transport error carries no
|
|
667
668
|
// `statusCode`); an operator can still uninstall while it's stopped,
|
|
668
669
|
// and `shutdown` then runs in this process, the only one available.
|
|
669
|
-
const daemon = await cliIpcCall<{
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
670
|
+
const daemon = await cliIpcCall<{
|
|
671
|
+
name: string;
|
|
672
|
+
target: string;
|
|
673
|
+
warnings?: PluginUninstallWarningKey[];
|
|
674
|
+
}>("plugins_uninstall", { pathParams: { name } });
|
|
675
|
+
let result: {
|
|
676
|
+
name: string;
|
|
677
|
+
target: string;
|
|
678
|
+
warnings?: PluginUninstallWarningKey[];
|
|
679
|
+
};
|
|
674
680
|
if (daemon.ok && daemon.result) {
|
|
675
681
|
result = daemon.result;
|
|
676
682
|
} else if (daemon.statusCode === undefined) {
|
|
@@ -693,6 +699,11 @@ export function registerPluginsCommand(program: Command): void {
|
|
|
693
699
|
console.log(
|
|
694
700
|
`Uninstalled plugin "${result.name}" from ${result.target}`,
|
|
695
701
|
);
|
|
702
|
+
for (const warning of result.warnings ?? []) {
|
|
703
|
+
console.warn(
|
|
704
|
+
`Warning: ${libs.uninstall.resolvePluginUninstallWarning(warning)}`,
|
|
705
|
+
);
|
|
706
|
+
}
|
|
696
707
|
} catch (err) {
|
|
697
708
|
if (err instanceof libs.installGitHub.InvalidPluginNameError) {
|
|
698
709
|
console.error(err.message);
|