@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/desktop-browser-cli.md +33 -0
- package/openapi.yaml +17 -5
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +279 -0
- package/src/__tests__/config-schema.test.ts +1 -1
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +6 -4
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/host-cu-proxy.test.ts +116 -18
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/mcp-auth-routes.test.ts +99 -0
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-health-check.test.ts +32 -32
- package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/skills.test.ts +10 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/apps/app-store.ts +6 -5
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/calls/__tests__/progress-narration.test.ts +23 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +7 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +62 -0
- package/src/calls/progress-narration.ts +15 -2
- package/src/calls/voice-control-protocol.ts +34 -4
- package/src/calls/voice-session-bridge.ts +23 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/plugins.ts +16 -5
- package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +13 -0
- package/src/cli/lib/install-from-github.ts +17 -18
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-details.ts +21 -8
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/bundled-skills/computer-use/SKILL.md +30 -0
- package/src/config/bundled-skills/computer-use/TOOLS.json +2 -2
- package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
- package/src/config/feature-flag-registry.json +2 -2
- package/src/config/schemas/__tests__/voice.test.ts +2 -2
- package/src/config/schemas/mcp.ts +9 -7
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/voice.ts +2 -2
- package/src/config/skills.ts +13 -21
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-runtime-assembly.ts +1 -1
- package/src/daemon/conversation-tool-setup.ts +31 -29
- package/src/daemon/host-cu-proxy.ts +51 -26
- package/src/daemon/host-proxy-preactivation.ts +2 -0
- package/src/daemon/mcp-reload-service.ts +7 -0
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/providers-setup.ts +4 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-automation-lease.test.ts +208 -0
- package/src/desktop/desktop-automation-lease.ts +270 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +58 -8
- package/src/desktop/desktop-dependencies.test.ts +29 -0
- package/src/desktop/desktop-dependencies.ts +26 -5
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.ts +14 -1
- package/src/desktop/desktop-session-manager.test.ts +86 -3
- package/src/desktop/desktop-session-manager.ts +144 -24
- package/src/desktop/desktop-stream-bridge.ts +1 -1
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/i18n/__tests__/i18n.test.ts +9 -5
- package/src/i18n/messages.ts +13 -4
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +22 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +3 -2
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/protocol.test.ts +11 -3
- package/src/live-voice/__tests__/session-controls.test.ts +34 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-session.ts +3 -0
- package/src/live-voice/protocol.ts +10 -1
- package/src/live-voice/session-controls.ts +28 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +11 -0
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +42 -15
- package/src/mcp/manager.ts +61 -8
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/oauth/seed-providers.ts +11 -0
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +9 -15
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
- package/src/providers/inference/endpoint-probe.ts +27 -12
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +167 -0
- package/src/providers/opencode/client.ts +73 -4
- package/src/runtime/AGENTS.md +4 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
- package/src/runtime/http-server.ts +6 -6
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
- package/src/runtime/routes/desktop-setup-routes.ts +7 -5
- package/src/runtime/routes/mcp-auth-routes.ts +106 -94
- package/src/runtime/routes/plugins-routes.ts +14 -1
- package/src/runtime/sync/resource-sync-events.ts +4 -0
- package/src/tools/browser/browser-execution.ts +44 -13
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/computer-use/definitions.ts +2 -3
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/plugin-manifest.ts +203 -0
- package/src/desktop/desktop-feature.test.ts +0 -18
- package/src/desktop/desktop-feature.ts +0 -19
|
@@ -540,8 +540,8 @@
|
|
|
540
540
|
"id": "assistant-desktop",
|
|
541
541
|
"scope": "assistant",
|
|
542
542
|
"key": "assistant-desktop",
|
|
543
|
-
"label": "
|
|
544
|
-
"description": "Streams an interactive desktop from a containerized assistant
|
|
543
|
+
"label": "Virtual desktop",
|
|
544
|
+
"description": "Streams an interactive virtual desktop from a platform-hosted, containerized assistant over /v1/desktop/stream. Requires platform hosting; self-hosted assistants are excluded. Off: the virtual desktop is hidden. On: the chat header offers its viewer and controls.",
|
|
545
545
|
"defaultEnabled": false
|
|
546
546
|
}
|
|
547
547
|
]
|
|
@@ -9,7 +9,7 @@ const PROGRESS_DEFAULTS = {
|
|
|
9
9
|
maxSilenceMs: 35_000,
|
|
10
10
|
longOpMs: 15_000,
|
|
11
11
|
minGapMs: 6_000,
|
|
12
|
-
generationTimeoutMs:
|
|
12
|
+
generationTimeoutMs: 5_000,
|
|
13
13
|
};
|
|
14
14
|
|
|
15
15
|
const FRONT_MODEL_DEFAULTS = {
|
|
@@ -75,7 +75,7 @@ describe("VoiceFrontModelConfigSchema", () => {
|
|
|
75
75
|
expect(parsed.progress.maxSilenceMs).toBe(35_000);
|
|
76
76
|
expect(parsed.progress.longOpMs).toBe(15_000);
|
|
77
77
|
expect(parsed.progress.minGapMs).toBe(6_000);
|
|
78
|
-
expect(parsed.progress.generationTimeoutMs).toBe(
|
|
78
|
+
expect(parsed.progress.generationTimeoutMs).toBe(5_000);
|
|
79
79
|
});
|
|
80
80
|
|
|
81
81
|
test("a stale maxPerTurn key is stripped, not rejected", () => {
|
|
@@ -55,9 +55,7 @@ const McpSseTransportSchema = z
|
|
|
55
55
|
.optional()
|
|
56
56
|
.describe("Custom HTTP headers sent with SSE requests"),
|
|
57
57
|
})
|
|
58
|
-
.describe(
|
|
59
|
-
"SSE transport: connects to an MCP server over Server-Sent Events",
|
|
60
|
-
);
|
|
58
|
+
.describe("SSE transport: connects to an MCP server over Server-Sent Events");
|
|
61
59
|
|
|
62
60
|
const McpStreamableHttpTransportSchema = z
|
|
63
61
|
.object({
|
|
@@ -114,10 +112,14 @@ export type McpConfig = z.infer<typeof McpConfigSchema>;
|
|
|
114
112
|
*/
|
|
115
113
|
export type McpServerSource = "workspace" | "plugin";
|
|
116
114
|
|
|
117
|
-
/** A server config with its origin
|
|
118
|
-
export
|
|
119
|
-
readonly source:
|
|
120
|
-
|
|
115
|
+
/** A server config with its runtime-only origin and credential identity. */
|
|
116
|
+
export type ResolvedMcpServerConfig =
|
|
117
|
+
| (McpServerConfig & { readonly source: "workspace" })
|
|
118
|
+
| (McpServerConfig & {
|
|
119
|
+
readonly source: "plugin";
|
|
120
|
+
readonly pluginName: string;
|
|
121
|
+
readonly serverKey: string;
|
|
122
|
+
});
|
|
121
123
|
|
|
122
124
|
/** The MCP config the daemon runs: both origins, every server attributed. */
|
|
123
125
|
export interface ResolvedMcpConfig {
|
|
@@ -75,6 +75,15 @@ export const MonitoringConfigSchema = z
|
|
|
75
75
|
.describe(
|
|
76
76
|
"Minimum interval between file-descriptor warnings while usage stays over the threshold, in milliseconds. Crossing the threshold always warns immediately; this only throttles the repeats.",
|
|
77
77
|
),
|
|
78
|
+
mountWatchIntervalMs: z
|
|
79
|
+
.number({ error: "monitoring.mountWatchIntervalMs must be a number" })
|
|
80
|
+
.int("monitoring.mountWatchIntervalMs must be an integer")
|
|
81
|
+
.min(1_000, "monitoring.mountWatchIntervalMs must be at least 1000ms")
|
|
82
|
+
.max(60_000, "monitoring.mountWatchIntervalMs must be <= 60000ms")
|
|
83
|
+
.default(10_000)
|
|
84
|
+
.describe(
|
|
85
|
+
"How often the monitor diffs the assistant container's mount table (/proc/self/mountinfo plus statfs of /workspace and /data), in milliseconds. Logs only when the watched mounts change. Default 10 seconds so a virtiofs bind drop is timestamped without a line per poll.",
|
|
86
|
+
),
|
|
78
87
|
pluginSourceScanIntervalMs: z
|
|
79
88
|
.number({
|
|
80
89
|
error: "monitoring.pluginSourceScanIntervalMs must be a number",
|
|
@@ -74,9 +74,9 @@ export const VoiceProgressConfigSchema = z
|
|
|
74
74
|
.positive(
|
|
75
75
|
"voice.frontModel.progress.generationTimeoutMs must be a positive integer",
|
|
76
76
|
)
|
|
77
|
-
.default(
|
|
77
|
+
.default(5_000)
|
|
78
78
|
.describe(
|
|
79
|
-
"Budget (ms) for LLM-generated progress text. Not latency-critical: it speaks into dead air",
|
|
79
|
+
"Budget (ms) for LLM-generated progress text. Not latency-critical: it speaks into dead air, so the budget is set to cover the narrator's real roundtrip rather than to be tight. A budget that only sometimes covers that roundtrip thins narration out rather than speeding it up, and it thins out exactly where the updates matter most, since the slow beats are the ones with the longest silence behind them (see endpointDecisionTimeoutMs for the same failure mode). Kept under progress.minGapMs so an update that spends the whole budget still lands before the next one is due",
|
|
80
80
|
),
|
|
81
81
|
})
|
|
82
82
|
// The heartbeat is checked when the idle tick finds the turn silent, so a
|
package/src/config/skills.ts
CHANGED
|
@@ -39,6 +39,10 @@ import {
|
|
|
39
39
|
getWorkspacePluginsDir,
|
|
40
40
|
getWorkspaceSkillsDir,
|
|
41
41
|
} from "../util/platform.js";
|
|
42
|
+
import {
|
|
43
|
+
hasPluginManifest,
|
|
44
|
+
readPluginManifest,
|
|
45
|
+
} from "../util/plugin-manifest.js";
|
|
42
46
|
import { stripCommentLines } from "../util/strip-comment-lines.js";
|
|
43
47
|
import { isAssistantFeatureFlagEnabled } from "./assistant-feature-flags.js";
|
|
44
48
|
import { getConfig } from "./loader.js";
|
|
@@ -734,8 +738,7 @@ function discoverSkillDirectories(skillsDir: string): string[] {
|
|
|
734
738
|
}
|
|
735
739
|
|
|
736
740
|
/**
|
|
737
|
-
* Whether `pluginDir` carries a plugin manifest the runtime can load
|
|
738
|
-
* parseable `package.json` with a non-empty string `name`. This mirrors the
|
|
741
|
+
* Whether `pluginDir` carries a plugin manifest the runtime can load. This mirrors the
|
|
739
742
|
* external plugin loader (`buildPluginFromDir`), which builds a plugin from
|
|
740
743
|
* any such directory and derives the plugin's identity from `package.json`
|
|
741
744
|
* `name` — it imposes no match between that `name` and the directory name.
|
|
@@ -745,31 +748,20 @@ function discoverSkillDirectories(skillsDir: string): string[] {
|
|
|
745
748
|
* requiring the two to match would silently drop the resident skills of every
|
|
746
749
|
* such plugin even though the runtime loads its hooks and tools fine.
|
|
747
750
|
*
|
|
748
|
-
* The caller is responsible for the missing
|
|
751
|
+
* The caller is responsible for the missing-manifest case (it emits a
|
|
749
752
|
* diagnostic warning); this function only judges a manifest that is present.
|
|
750
753
|
*/
|
|
751
754
|
function hasLoadablePluginManifest(pluginDir: string): boolean {
|
|
752
|
-
const manifestPath = join(pluginDir, "package.json");
|
|
753
|
-
if (!existsSync(manifestPath)) {
|
|
754
|
-
return false;
|
|
755
|
-
}
|
|
756
|
-
let parsed: unknown;
|
|
757
755
|
try {
|
|
758
|
-
|
|
756
|
+
readPluginManifest(pluginDir);
|
|
757
|
+
return true;
|
|
759
758
|
} catch (err) {
|
|
760
759
|
log.warn(
|
|
761
|
-
{ err,
|
|
762
|
-
"Skipping plugin dir with
|
|
760
|
+
{ err, pluginDir },
|
|
761
|
+
"Skipping plugin dir with an invalid manifest for resident skills",
|
|
763
762
|
);
|
|
764
763
|
return false;
|
|
765
764
|
}
|
|
766
|
-
return (
|
|
767
|
-
typeof parsed === "object" &&
|
|
768
|
-
parsed !== null &&
|
|
769
|
-
"name" in parsed &&
|
|
770
|
-
typeof (parsed as { name: unknown }).name === "string" &&
|
|
771
|
-
(parsed as { name: string }).name.length > 0
|
|
772
|
-
);
|
|
773
765
|
}
|
|
774
766
|
|
|
775
767
|
/**
|
|
@@ -885,15 +877,15 @@ function discoverInstalledPluginResidentSkills(): SkillSummary[] {
|
|
|
885
877
|
}
|
|
886
878
|
const pluginDir = join(pluginsDir, entry.name);
|
|
887
879
|
|
|
888
|
-
// A directory under `plugins/` with no
|
|
880
|
+
// A directory under `plugins/` with no supported manifest is not a plugin the
|
|
889
881
|
// runtime can load, so its skills are never surfaced. This is an easy
|
|
890
882
|
// footgun — a plugin dropped in without its manifest looks installed but
|
|
891
883
|
// silently contributes nothing — so warn loudly with the path rather than
|
|
892
884
|
// skipping in silence, to make the misconfiguration diagnosable.
|
|
893
|
-
if (!
|
|
885
|
+
if (!hasPluginManifest(pluginDir)) {
|
|
894
886
|
log.warn(
|
|
895
887
|
{ pluginDir },
|
|
896
|
-
"Plugin directory is missing package.json
|
|
888
|
+
"Plugin directory is missing package.json and plugin.json; its skills will not be available.",
|
|
897
889
|
);
|
|
898
890
|
continue;
|
|
899
891
|
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
|
|
8
8
|
|
|
9
|
+
import { setOverridesForTesting } from "../../__tests__/feature-flag-test-helpers.js";
|
|
9
10
|
import * as configLoader from "../../config/loader.js";
|
|
10
11
|
import type { AssistantConfig } from "../../config/schema.js";
|
|
11
12
|
import * as disabledState from "../../plugins/disabled-state.js";
|
|
@@ -245,3 +246,49 @@ describe("createResolveToolsCallback — config.tools.exclude", () => {
|
|
|
245
246
|
expect(ctx.allowedToolNames?.has("bash")).toBe(false);
|
|
246
247
|
});
|
|
247
248
|
});
|
|
249
|
+
|
|
250
|
+
test("managed browser guidance follows each turn without changing native or shared tool definitions", () => {
|
|
251
|
+
const platform = process.env.IS_PLATFORM;
|
|
252
|
+
const containerized = process.env.IS_CONTAINERIZED;
|
|
253
|
+
process.env.IS_PLATFORM = "true";
|
|
254
|
+
process.env.IS_CONTAINERIZED = "true";
|
|
255
|
+
setOverridesForTesting({ "assistant-desktop": true });
|
|
256
|
+
const bash = def("bash");
|
|
257
|
+
const ctx = makeCtx({
|
|
258
|
+
conversationId: "conv-browser-guidance",
|
|
259
|
+
transportInterface: "web",
|
|
260
|
+
currentTurnClientOs: "web",
|
|
261
|
+
trustContext: { trustClass: "guardian" } as Conversation["trustContext"],
|
|
262
|
+
getTurnActorPrincipalId: () => "user-123",
|
|
263
|
+
});
|
|
264
|
+
const resolve = createResolveToolsCallback([bash], ctx)!;
|
|
265
|
+
try {
|
|
266
|
+
expect(
|
|
267
|
+
resolve([]).find((tool) => tool.name === "bash")?.description,
|
|
268
|
+
).toContain("assistant browser navigate");
|
|
269
|
+
expect(bash.description).toBe("bash");
|
|
270
|
+
for (const clientOs of ["macos", "windows", "linux"] as const) {
|
|
271
|
+
ctx.currentTurnClientOs = clientOs;
|
|
272
|
+
expect(
|
|
273
|
+
resolve([]).find((tool) => tool.name === "bash")?.description,
|
|
274
|
+
).toBe("bash");
|
|
275
|
+
}
|
|
276
|
+
ctx.currentTurnClientOs = "web";
|
|
277
|
+
setOverridesForTesting({ "assistant-desktop": false });
|
|
278
|
+
expect(resolve([]).find((tool) => tool.name === "bash")?.description).toBe(
|
|
279
|
+
"bash",
|
|
280
|
+
);
|
|
281
|
+
} finally {
|
|
282
|
+
setOverridesForTesting({});
|
|
283
|
+
if (platform === undefined) {
|
|
284
|
+
delete process.env.IS_PLATFORM;
|
|
285
|
+
} else {
|
|
286
|
+
process.env.IS_PLATFORM = platform;
|
|
287
|
+
}
|
|
288
|
+
if (containerized === undefined) {
|
|
289
|
+
delete process.env.IS_CONTAINERIZED;
|
|
290
|
+
} else {
|
|
291
|
+
process.env.IS_CONTAINERIZED = containerized;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type ClientOs, parseClientOs } from "../channels/types.js";
|
|
2
|
+
import type { Conversation } from "./conversation.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Host OS of the client driving this turn. The Electron renderer reports
|
|
6
|
+
* `interface: "web"` and carries the real OS in `clientOs`, so this prefers
|
|
7
|
+
* the frozen per-turn value and only falls back to a desktop transport.
|
|
8
|
+
*/
|
|
9
|
+
export function resolveTurnClientOs(ctx: Conversation): {
|
|
10
|
+
clientOs: ClientOs | undefined;
|
|
11
|
+
transportInterface: Conversation["transportInterface"];
|
|
12
|
+
} {
|
|
13
|
+
const pin = ctx.toolContextPin;
|
|
14
|
+
const transportInterface = pin
|
|
15
|
+
? pin.transportInterface
|
|
16
|
+
: ctx.transportInterface;
|
|
17
|
+
const clientOs = pin
|
|
18
|
+
? pin.clientOs
|
|
19
|
+
: (parseClientOs(ctx.currentTurnClientOs ?? ctx.clientOs) ??
|
|
20
|
+
(transportInterface === "macos" ||
|
|
21
|
+
transportInterface === "windows" ||
|
|
22
|
+
transportInterface === "linux"
|
|
23
|
+
? transportInterface
|
|
24
|
+
: undefined));
|
|
25
|
+
return { clientOs, transportInterface };
|
|
26
|
+
}
|
|
@@ -854,7 +854,7 @@ export function buildChannelCapabilityBlock(
|
|
|
854
854
|
if (clientOs === "macos") {
|
|
855
855
|
lines.push("");
|
|
856
856
|
lines.push(
|
|
857
|
-
"On macOS,
|
|
857
|
+
"On macOS, drive apps with the computer-use skill; `host_bash` is for shell commands.",
|
|
858
858
|
);
|
|
859
859
|
}
|
|
860
860
|
|
|
@@ -7,10 +7,9 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import type { AssistantEvent } from "../api/index.js";
|
|
10
|
+
import { shouldUseVirtualDesktopBrowser } from "../browser/virtual-desktop-target.js";
|
|
10
11
|
import {
|
|
11
|
-
type ClientOs,
|
|
12
12
|
type HostProxyCapability,
|
|
13
|
-
parseClientOs,
|
|
14
13
|
supportsHostProxy,
|
|
15
14
|
} from "../channels/types.js";
|
|
16
15
|
import { getIsPlatform } from "../config/env-registry.js";
|
|
@@ -74,6 +73,7 @@ import {
|
|
|
74
73
|
conversationSupportsGuardianQuestionCards,
|
|
75
74
|
} from "./channel-ui-capability.js";
|
|
76
75
|
import type { Conversation } from "./conversation.js";
|
|
76
|
+
import { resolveTurnClientOs } from "./conversation-client-surface.js";
|
|
77
77
|
import { projectSkillTools } from "./conversation-skill-tools.js";
|
|
78
78
|
import {
|
|
79
79
|
restoreSurfaceStateEntry,
|
|
@@ -742,30 +742,6 @@ export const ALLOWLIST_ONLY_TOOL_NAMES = new Set<string>([
|
|
|
742
742
|
"delete_memory_page",
|
|
743
743
|
]);
|
|
744
744
|
|
|
745
|
-
/**
|
|
746
|
-
* Host OS of the client driving this turn. The Electron renderer reports
|
|
747
|
-
* `interface: "web"` and carries the real OS in `clientOs`, so this prefers
|
|
748
|
-
* the frozen per-turn value and only falls back to a desktop transport.
|
|
749
|
-
*/
|
|
750
|
-
function resolveTurnClientOs(ctx: Conversation): {
|
|
751
|
-
clientOs: ClientOs | undefined;
|
|
752
|
-
transportInterface: Conversation["transportInterface"];
|
|
753
|
-
} {
|
|
754
|
-
const pin = ctx.toolContextPin;
|
|
755
|
-
const transportInterface = pin
|
|
756
|
-
? pin.transportInterface
|
|
757
|
-
: ctx.transportInterface;
|
|
758
|
-
const clientOs = pin
|
|
759
|
-
? pin.clientOs
|
|
760
|
-
: (parseClientOs(ctx.currentTurnClientOs ?? ctx.clientOs) ??
|
|
761
|
-
(transportInterface === "macos" ||
|
|
762
|
-
transportInterface === "windows" ||
|
|
763
|
-
transportInterface === "linux"
|
|
764
|
-
? transportInterface
|
|
765
|
-
: undefined));
|
|
766
|
-
return { clientOs, transportInterface };
|
|
767
|
-
}
|
|
768
|
-
|
|
769
745
|
/**
|
|
770
746
|
* Windows parity gate: skill tools may declare `supported_client_os`; drop
|
|
771
747
|
* them when the turn's client OS (or pinned OS for wakes) is not listed.
|
|
@@ -1149,9 +1125,35 @@ export function createResolveToolsCallback(
|
|
|
1149
1125
|
// while background calls persist the full surface content for the next
|
|
1150
1126
|
// capable client that opens the conversation. Skill tools stay off this
|
|
1151
1127
|
// list (`skill_execute` dispatch) and are a separate disclosure path.
|
|
1152
|
-
let allBaseDefs = [
|
|
1153
|
-
|
|
1154
|
-
|
|
1128
|
+
let allBaseDefs = [
|
|
1129
|
+
...scopedCoreDefs,
|
|
1130
|
+
...scopedWorkspaceDefs,
|
|
1131
|
+
...scopedMcpDefs,
|
|
1132
|
+
].filter((d) => !excluded.has(d.name));
|
|
1133
|
+
if (
|
|
1134
|
+
ctx.transportInterface === "web" &&
|
|
1135
|
+
shouldUseVirtualDesktopBrowser(
|
|
1136
|
+
undefined,
|
|
1137
|
+
{},
|
|
1138
|
+
{
|
|
1139
|
+
workingDir: ctx.workingDir,
|
|
1140
|
+
conversationId: ctx.conversationId,
|
|
1141
|
+
trustClass: ctx.trustContext?.trustClass ?? "unknown",
|
|
1142
|
+
transportInterface: ctx.transportInterface,
|
|
1143
|
+
clientOs: resolveTurnClientOs(ctx).clientOs,
|
|
1144
|
+
sourceActorPrincipalId: ctx.getTurnActorPrincipalId?.(),
|
|
1145
|
+
},
|
|
1146
|
+
)
|
|
1147
|
+
) {
|
|
1148
|
+
allBaseDefs = allBaseDefs.map((definition) =>
|
|
1149
|
+
definition.name === "bash"
|
|
1150
|
+
? {
|
|
1151
|
+
...definition,
|
|
1152
|
+
description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. It installs the virtual desktop if needed, starts Chrome, and completes the action in one call. Use timeout_seconds: ${getConfig().timeouts.shellMaxTimeoutSec} for first use; setup progress is visible in the Virtual desktop panel. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
|
|
1153
|
+
}
|
|
1154
|
+
: definition,
|
|
1155
|
+
);
|
|
1156
|
+
}
|
|
1155
1157
|
// Activation-rail conversations carry the optional `activation_moment`
|
|
1156
1158
|
// telemetry param on ui_show. The marker is written before the first
|
|
1157
1159
|
// tool resolution (see `applyBootstrapTemplate` in system-prompt.ts), so
|
|
@@ -48,8 +48,8 @@ const SCREENSHOT_OMITTED_MESSAGE =
|
|
|
48
48
|
"Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
|
|
49
49
|
|
|
50
50
|
// computer_use_key combos that change only selection/cursor/clipboard state.
|
|
51
|
-
// The AX tree models none of these, so they always produce an empty diff
|
|
52
|
-
// exempt
|
|
51
|
+
// The AX tree models none of these, so they always produce an empty diff and
|
|
52
|
+
// are exempt from the unchanged-tree signal.
|
|
53
53
|
// Stored in canonical form (see canonicalizeKeyCombo): modifier aliases
|
|
54
54
|
// normalized and ordered, so `cmd + a`, `command+a`, `alt+tab`, `tab+shift`
|
|
55
55
|
// all match.
|
|
@@ -129,12 +129,28 @@ export interface ActionRecord {
|
|
|
129
129
|
reasoning?: string;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
+
// Steps whose empty diff says nothing about whether they worked: waiting and
|
|
133
|
+
// observing change nothing by design, and an AppleScript reports its own
|
|
134
|
+
// outcome in the execution result, often for an app whose content the tree
|
|
135
|
+
// cannot see. They neither warn nor move the unchanged streak.
|
|
136
|
+
const NO_AX_DIFF_TOOLS = new Set([
|
|
137
|
+
"computer_use_wait",
|
|
138
|
+
"computer_use_observe",
|
|
139
|
+
"computer_use_run_applescript",
|
|
140
|
+
]);
|
|
141
|
+
|
|
142
|
+
type ActionIdentity = Pick<ActionRecord, "toolName" | "input">;
|
|
143
|
+
|
|
144
|
+
function isNoDiffTool(action: ActionIdentity | undefined): boolean {
|
|
145
|
+
return action !== undefined && NO_AX_DIFF_TOOLS.has(action.toolName);
|
|
146
|
+
}
|
|
147
|
+
|
|
132
148
|
/**
|
|
133
149
|
* True when `action` is a computer_use_key press whose key only mutates
|
|
134
|
-
* selection/cursor/clipboard state
|
|
150
|
+
* selection/cursor/clipboard state: changes the AX tree cannot represent, so
|
|
135
151
|
* an empty diff is expected rather than a sign the action did nothing.
|
|
136
152
|
*/
|
|
137
|
-
function isNoDiffKeyAction(action:
|
|
153
|
+
function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
|
|
138
154
|
if (action?.toolName !== "computer_use_key") {
|
|
139
155
|
return false;
|
|
140
156
|
}
|
|
@@ -245,6 +261,7 @@ export class HostCuProxy {
|
|
|
245
261
|
resetGeneration: number;
|
|
246
262
|
dispatchedAt: number;
|
|
247
263
|
toolName: string;
|
|
264
|
+
input: Record<string, unknown>;
|
|
248
265
|
step: number;
|
|
249
266
|
}
|
|
250
267
|
>();
|
|
@@ -455,6 +472,7 @@ export class HostCuProxy {
|
|
|
455
472
|
resetGeneration: this._resetGeneration,
|
|
456
473
|
dispatchedAt: Date.now(),
|
|
457
474
|
toolName,
|
|
475
|
+
input,
|
|
458
476
|
step: stepNumber,
|
|
459
477
|
});
|
|
460
478
|
|
|
@@ -538,7 +556,7 @@ export class HostCuProxy {
|
|
|
538
556
|
}
|
|
539
557
|
|
|
540
558
|
// A targeted snapshot has no comparable action/diff baseline; neither it
|
|
541
|
-
// nor the first desktop observation after it can imply
|
|
559
|
+
// nor the first desktop observation after it can imply an unchanged tree.
|
|
542
560
|
// A response dispatched before the last reset belongs to a finished run,
|
|
543
561
|
// so it must not change any state the new run has started building.
|
|
544
562
|
const fromCurrentRun =
|
|
@@ -548,6 +566,11 @@ export class HostCuProxy {
|
|
|
548
566
|
this._consecutiveUnchangedSteps = 0;
|
|
549
567
|
}
|
|
550
568
|
const prevAXTree = this._previousAXTree;
|
|
569
|
+
// Judge the empty diff against the action this request carried, for the
|
|
570
|
+
// same reason the timings line does.
|
|
571
|
+
const action: ActionIdentity | undefined = owned
|
|
572
|
+
? { toolName: owned.toolName, input: owned.input }
|
|
573
|
+
: this.lastRecordedAction();
|
|
551
574
|
const comparableObservation = scopedObservation
|
|
552
575
|
? { ...observation, axDiff: undefined, secondaryWindows: undefined }
|
|
553
576
|
: observation;
|
|
@@ -557,7 +580,7 @@ export class HostCuProxy {
|
|
|
557
580
|
!scopedObservation &&
|
|
558
581
|
owned?.toolName !== POINT_AT_PROXY_TOOL
|
|
559
582
|
) {
|
|
560
|
-
this.updateStateFromObservation(comparableObservation);
|
|
583
|
+
this.updateStateFromObservation(comparableObservation, action);
|
|
561
584
|
// A desktop has had its first look only once pixels from it arrived,
|
|
562
585
|
// so a failed capture leaves the next request asking again.
|
|
563
586
|
if (owned && observation.screenshot) {
|
|
@@ -568,6 +591,7 @@ export class HostCuProxy {
|
|
|
568
591
|
comparableObservation,
|
|
569
592
|
prevAXTree,
|
|
570
593
|
owned?.screenshotSkipped ?? false,
|
|
594
|
+
action,
|
|
571
595
|
);
|
|
572
596
|
interaction.rpcResolve(result);
|
|
573
597
|
return result;
|
|
@@ -672,6 +696,7 @@ export class HostCuProxy {
|
|
|
672
696
|
obs: CuObservationResult,
|
|
673
697
|
previousAXTree?: string,
|
|
674
698
|
screenshotSkipped = false,
|
|
699
|
+
action: ActionIdentity | undefined = this.lastRecordedAction(),
|
|
675
700
|
): ToolExecutionResult {
|
|
676
701
|
const prevTree = previousAXTree;
|
|
677
702
|
const parts: string[] = [];
|
|
@@ -690,24 +715,17 @@ export class HostCuProxy {
|
|
|
690
715
|
parts.push(obs.axDiff);
|
|
691
716
|
parts.push("");
|
|
692
717
|
} else if (prevTree != null && obs.axTree != null) {
|
|
693
|
-
|
|
694
|
-
this._actionHistory.length > 0
|
|
695
|
-
? this._actionHistory[this._actionHistory.length - 1]
|
|
696
|
-
: undefined;
|
|
697
|
-
const isWaitAction = lastAction?.toolName === "computer_use_wait";
|
|
698
|
-
const isNoDiffKey = isNoDiffKeyAction(lastAction);
|
|
699
|
-
|
|
700
|
-
if (!isWaitAction && !isNoDiffKey) {
|
|
718
|
+
if (!isNoDiffTool(action) && !isNoDiffKeyAction(action)) {
|
|
701
719
|
if (
|
|
702
720
|
this._consecutiveUnchangedSteps >=
|
|
703
721
|
CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
|
|
704
722
|
) {
|
|
705
723
|
parts.push(
|
|
706
|
-
`WARNING: ${this._consecutiveUnchangedSteps} consecutive actions
|
|
724
|
+
`WARNING: the accessibility tree did not change across ${this._consecutiveUnchangedSteps} consecutive actions. If the screenshot does not show them working either, try a different approach.`,
|
|
707
725
|
);
|
|
708
726
|
} else {
|
|
709
727
|
parts.push(
|
|
710
|
-
"
|
|
728
|
+
"The accessibility tree did not change after your last action. Apps that draw their own content (timelines, canvases, spreadsheet grids) change without it, so check the screenshot before deciding the action did nothing.",
|
|
711
729
|
);
|
|
712
730
|
}
|
|
713
731
|
parts.push("");
|
|
@@ -822,19 +840,26 @@ export class HostCuProxy {
|
|
|
822
840
|
// Private helpers
|
|
823
841
|
// ---------------------------------------------------------------------------
|
|
824
842
|
|
|
825
|
-
private
|
|
843
|
+
private lastRecordedAction(): ActionRecord | undefined {
|
|
844
|
+
return this._actionHistory.at(-1);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
private updateStateFromObservation(
|
|
848
|
+
obs: CuObservationResult,
|
|
849
|
+
action: ActionIdentity | undefined,
|
|
850
|
+
): void {
|
|
826
851
|
if (this._stepCount > 0) {
|
|
827
|
-
|
|
828
|
-
this._actionHistory.length > 0
|
|
829
|
-
? this._actionHistory[this._actionHistory.length - 1]
|
|
830
|
-
: undefined;
|
|
831
|
-
if (obs.axDiff != null || isNoDiffKeyAction(lastAction)) {
|
|
852
|
+
if (obs.axDiff != null || isNoDiffKeyAction(action)) {
|
|
832
853
|
// A real diff, or an exempt key whose effect is invisible by design,
|
|
833
|
-
// breaks the no-effect streak
|
|
834
|
-
// stale count
|
|
835
|
-
// into a false "consecutive" escalation.
|
|
854
|
+
// breaks the no-effect streak. Clearing it rather than preserving a
|
|
855
|
+
// stale count keeps an intervening cmd+a from bridging two no-op
|
|
856
|
+
// actions into a false "consecutive" escalation.
|
|
836
857
|
this._consecutiveUnchangedSteps = 0;
|
|
837
|
-
} else if (
|
|
858
|
+
} else if (
|
|
859
|
+
!isNoDiffTool(action) &&
|
|
860
|
+
this._previousAXTree != null &&
|
|
861
|
+
obs.axTree != null
|
|
862
|
+
) {
|
|
838
863
|
this._consecutiveUnchangedSteps++;
|
|
839
864
|
}
|
|
840
865
|
}
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
* - `daemon/process-message.ts` (create path, prepareConversationForMessage)
|
|
10
10
|
* - `daemon/conversation-process.ts` `drainSingleMessage` (re-add after dequeue)
|
|
11
11
|
* - `daemon/conversation-process.ts` `drainBatch` (re-add after dequeue)
|
|
12
|
+
* - `calls/voice-session-bridge.ts` (tool-capable legs of a live-voice
|
|
13
|
+
* session opened from the macOS desktop client)
|
|
12
14
|
*
|
|
13
15
|
* The create paths additionally instantiate the proxy itself; that
|
|
14
16
|
* instantiation logic is per-proxy-class and stays inline at each create
|
|
@@ -14,6 +14,7 @@ import { getMcpServerManager } from "../mcp/manager.js";
|
|
|
14
14
|
import { migrateLegacyMcpHeaders } from "../mcp/mcp-header-store.js";
|
|
15
15
|
import { signalMcpReloaded } from "../mcp/reload-signal.js";
|
|
16
16
|
import { loadWorkspaceMcpConfig } from "../mcp/workspace-mcp-config.js";
|
|
17
|
+
import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
|
|
17
18
|
import { createMcpToolsFromServer } from "../tools/mcp/mcp-tool-factory.js";
|
|
18
19
|
import { registerMcpTools, unregisterAllMcpTools } from "../tools/registry.js";
|
|
19
20
|
import { getLogger } from "../util/logger.js";
|
|
@@ -80,6 +81,7 @@ export async function reconcilePluginMcpServers(): Promise<void> {
|
|
|
80
81
|
}
|
|
81
82
|
|
|
82
83
|
async function doReload(): Promise<McpReloadResult> {
|
|
84
|
+
let teardownStarted = false;
|
|
83
85
|
try {
|
|
84
86
|
const manager = getMcpServerManager();
|
|
85
87
|
|
|
@@ -98,6 +100,7 @@ async function doReload(): Promise<McpReloadResult> {
|
|
|
98
100
|
invalidateConfigCache();
|
|
99
101
|
|
|
100
102
|
// 2. Stop existing MCP servers + unregister their tools
|
|
103
|
+
teardownStarted = true;
|
|
101
104
|
await manager.stop();
|
|
102
105
|
unregisterAllMcpTools();
|
|
103
106
|
|
|
@@ -160,5 +163,9 @@ async function doReload(): Promise<McpReloadResult> {
|
|
|
160
163
|
const error = err instanceof Error ? err.message : String(err);
|
|
161
164
|
log.error({ err }, "MCP reload failed");
|
|
162
165
|
return { success: false, error };
|
|
166
|
+
} finally {
|
|
167
|
+
if (teardownStarted) {
|
|
168
|
+
publishMcpChanged();
|
|
169
|
+
}
|
|
163
170
|
}
|
|
164
171
|
}
|
|
@@ -18,6 +18,7 @@ export const SYNC_TAGS = {
|
|
|
18
18
|
appsList: "apps:list",
|
|
19
19
|
documentsList: "documents:list",
|
|
20
20
|
pluginsList: "plugins:list",
|
|
21
|
+
mcpList: "mcp:list",
|
|
21
22
|
conversationsList: "conversations:list",
|
|
22
23
|
/** Activation-checklist progress: task launches, step counts, completions. */
|
|
23
24
|
activationProgress: "activation:progress",
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { maybeDefaultSpeechToManaged } from "../config/managed-speech-defaults.js";
|
|
2
2
|
import { rehydratePlatformCredentials } from "../config/platform-rehydration.js";
|
|
3
3
|
import type { AssistantConfig } from "../config/types.js";
|
|
4
|
+
import { getMcpServerManager } from "../mcp/manager.js";
|
|
4
5
|
import { startConfiguredMcpServers } from "../mcp/startup.js";
|
|
5
6
|
import { gmailMessagingProvider } from "../messaging/providers/gmail/adapter.js";
|
|
6
7
|
import { outlookMessagingProvider } from "../messaging/providers/outlook/adapter.js";
|
|
@@ -9,6 +10,7 @@ import { telegramBotMessagingProvider } from "../messaging/providers/telegram-bo
|
|
|
9
10
|
import { whatsappMessagingProvider } from "../messaging/providers/whatsapp/adapter.js";
|
|
10
11
|
import { registerMessagingProvider } from "../messaging/registry.js";
|
|
11
12
|
import { initializeProviders } from "../providers/registry.js";
|
|
13
|
+
import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
|
|
12
14
|
import { validateSubagentRoleAllowlists } from "../subagent/validate-allowlists.js";
|
|
13
15
|
import { initializeTools } from "../tools/registry.js";
|
|
14
16
|
import { getLogger } from "../util/logger.js";
|
|
@@ -63,7 +65,9 @@ export async function initializeProvidersAndTools(
|
|
|
63
65
|
// Start MCP servers — workspace-configured and plugin-declared alike —
|
|
64
66
|
// and register their tools. Shared with the schedule worker, which hosts
|
|
65
67
|
// agent turns in its own process and so needs its own connections.
|
|
68
|
+
getMcpServerManager().setUnexpectedCloseHandler(() => publishMcpChanged());
|
|
66
69
|
await startConfiguredMcpServers();
|
|
70
|
+
publishMcpChanged();
|
|
67
71
|
|
|
68
72
|
log.info("Daemon startup: providers and tools initialized");
|
|
69
73
|
}
|
|
@@ -58,6 +58,7 @@ export function newFakeDesktop(options: FakeDesktopOptions) {
|
|
|
58
58
|
let vncReady = true;
|
|
59
59
|
let chromiumPath: () => Promise<string> = async () => "/fake/chromium";
|
|
60
60
|
const manager = new DesktopSessionManager({
|
|
61
|
+
allocateDebugPort: async () => 9222,
|
|
61
62
|
spawn: (role, request) => {
|
|
62
63
|
if (options.failSpawn?.includes(role)) {
|
|
63
64
|
throw new Error(`spawn ${role} failed`);
|