@vellumai/assistant 0.12.2-staging.1 → 0.12.2-staging.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/docs/desktop-browser-cli.md +33 -0
  2. package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
  3. package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
  4. package/openapi.yaml +50 -4
  5. package/package.json +1 -1
  6. package/scripts/smoke-desktop-browser-cli.ts +279 -0
  7. package/src/__tests__/always-loaded-tools-guard.test.ts +5 -5
  8. package/src/__tests__/config-schema.test.ts +1 -1
  9. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  10. package/src/__tests__/conversation-error.test.ts +4 -1
  11. package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
  12. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
  13. package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
  14. package/src/__tests__/credential-routes.test.ts +38 -0
  15. package/src/__tests__/credential-security-invariants.test.ts +1 -1
  16. package/src/__tests__/cu-unified-flow.test.ts +6 -2
  17. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  18. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  19. package/src/__tests__/host-cu-proxy.test.ts +185 -30
  20. package/src/__tests__/list-all-apps.test.ts +40 -8
  21. package/src/__tests__/mcp-auth-routes.test.ts +99 -0
  22. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  23. package/src/__tests__/mcp-health-check.test.ts +32 -32
  24. package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
  25. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  26. package/src/__tests__/oauth-commands-routes.test.ts +55 -0
  27. package/src/__tests__/oauth-provider-serializer.test.ts +22 -0
  28. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  29. package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
  30. package/src/__tests__/skills.test.ts +10 -0
  31. package/src/__tests__/subagent-tool-gate-mode.test.ts +51 -0
  32. package/src/__tests__/ui-channel-variants.test.ts +1 -56
  33. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  34. package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
  35. package/src/acp/__tests__/acp-credentials.test.ts +13 -0
  36. package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
  37. package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
  38. package/src/acp/acp-claude-oauth.ts +328 -14
  39. package/src/acp/acp-credentials.ts +19 -0
  40. package/src/acp/claude-token-refresh.ts +150 -0
  41. package/src/acp/prepare-agent-env.ts +21 -6
  42. package/src/apps/app-store.ts +6 -5
  43. package/src/browser/operations.ts +2 -1
  44. package/src/browser/types.ts +7 -0
  45. package/src/browser/virtual-desktop-target.ts +34 -0
  46. package/src/calls/__tests__/progress-narration.test.ts +23 -0
  47. package/src/calls/__tests__/voice-control-protocol.test.ts +69 -0
  48. package/src/calls/__tests__/voice-session-bridge.test.ts +81 -0
  49. package/src/calls/progress-narration.ts +15 -2
  50. package/src/calls/voice-control-protocol.ts +132 -0
  51. package/src/calls/voice-session-bridge.ts +56 -26
  52. package/src/calls/voice-triage-escalate.ts +3 -3
  53. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  54. package/src/cli/commands/browser.help.ts +43 -0
  55. package/src/cli/commands/browser.ts +52 -8
  56. package/src/cli/commands/oauth/status.ts +5 -0
  57. package/src/cli/commands/plugins.ts +16 -5
  58. package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
  59. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  60. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  61. package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
  62. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  63. package/src/cli/lib/bundled-marketplace.json +13 -0
  64. package/src/cli/lib/install-from-github.ts +17 -18
  65. package/src/cli/lib/install-from-platform.ts +14 -9
  66. package/src/cli/lib/list-installed-plugins.ts +21 -35
  67. package/src/cli/lib/plugin-details.ts +21 -8
  68. package/src/cli/lib/uninstall-plugin.ts +59 -1
  69. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  70. package/src/config/bundled-skills/computer-use/SKILL.md +43 -4
  71. package/src/config/bundled-skills/computer-use/TOOLS.json +3 -3
  72. package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
  73. package/src/config/feature-flag-registry.json +11 -3
  74. package/src/config/loader.ts +1 -0
  75. package/src/config/schemas/__tests__/voice.test.ts +2 -2
  76. package/src/config/schemas/mcp.ts +9 -7
  77. package/src/config/schemas/monitoring.ts +9 -0
  78. package/src/config/schemas/services.ts +10 -0
  79. package/src/config/schemas/voice.ts +2 -2
  80. package/src/config/skills.ts +13 -21
  81. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  82. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
  83. package/src/daemon/conversation-client-surface.ts +26 -0
  84. package/src/daemon/conversation-runtime-assembly.ts +19 -7
  85. package/src/daemon/conversation-surfaces.ts +6 -1
  86. package/src/daemon/conversation-tool-setup.ts +37 -43
  87. package/src/daemon/host-cu-proxy.ts +94 -31
  88. package/src/daemon/host-proxy-preactivation.ts +2 -0
  89. package/src/daemon/mcp-reload-service.ts +7 -0
  90. package/src/daemon/message-types/sync.ts +1 -0
  91. package/src/daemon/providers-setup.ts +4 -0
  92. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  93. package/src/desktop/desktop-automation-lease.test.ts +208 -0
  94. package/src/desktop/desktop-automation-lease.ts +270 -0
  95. package/src/desktop/desktop-browser-client.test.ts +413 -0
  96. package/src/desktop/desktop-browser-client.ts +430 -0
  97. package/src/desktop/desktop-browser-cursor.ts +40 -0
  98. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  99. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  100. package/src/desktop/desktop-browser-operations.ts +130 -0
  101. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  102. package/src/desktop/desktop-chrome-session.ts +58 -8
  103. package/src/desktop/desktop-dependencies.test.ts +29 -0
  104. package/src/desktop/desktop-dependencies.ts +26 -5
  105. package/src/desktop/desktop-display.ts +9 -0
  106. package/src/desktop/desktop-panel-config.ts +14 -1
  107. package/src/desktop/desktop-session-manager.test.ts +86 -3
  108. package/src/desktop/desktop-session-manager.ts +144 -24
  109. package/src/desktop/desktop-stream-bridge.ts +1 -1
  110. package/src/desktop/virtual-desktop-feature.ts +18 -0
  111. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  112. package/src/i18n/__tests__/i18n.test.ts +9 -5
  113. package/src/i18n/messages.ts +13 -4
  114. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  115. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  116. package/src/ipc/assistant-server.ts +15 -1
  117. package/src/ipc/cli-client.ts +13 -5
  118. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
  119. package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
  120. package/src/live-voice/__tests__/live-voice-progress.test.ts +79 -0
  121. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  122. package/src/live-voice/__tests__/protocol.test.ts +47 -0
  123. package/src/live-voice/__tests__/session-controls.test.ts +113 -0
  124. package/src/live-voice/live-voice-metrics.ts +9 -0
  125. package/src/live-voice/live-voice-session.ts +88 -7
  126. package/src/live-voice/protocol.ts +69 -0
  127. package/src/live-voice/session-controls.ts +139 -0
  128. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  129. package/src/mcp/__tests__/effective-config.test.ts +11 -0
  130. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  131. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  132. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  133. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  134. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  135. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  136. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  137. package/src/mcp/client.ts +64 -16
  138. package/src/mcp/credential-target.ts +105 -0
  139. package/src/mcp/effective-config.ts +42 -15
  140. package/src/mcp/manager.ts +61 -8
  141. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  142. package/src/mcp/mcp-auth-state.ts +38 -6
  143. package/src/mcp/mcp-oauth-provider.ts +101 -33
  144. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  145. package/src/monitoring/mount-watch.ts +348 -0
  146. package/src/monitoring/worker.ts +9 -0
  147. package/src/oauth/__tests__/seed-providers-managed.test.ts +95 -0
  148. package/src/oauth/connection-resolver.test.ts +27 -0
  149. package/src/oauth/connection-resolver.ts +25 -1
  150. package/src/oauth/provider-serializer.ts +9 -0
  151. package/src/oauth/seed-providers.ts +121 -2
  152. package/src/onboarding/checkin-event.ts +2 -1
  153. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  154. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  155. package/src/plugins/external-plugin-loader.ts +49 -86
  156. package/src/plugins/installed-plugin-dirs.ts +6 -4
  157. package/src/plugins/mcp-servers.ts +9 -15
  158. package/src/plugins/mtime-cache.ts +2 -1
  159. package/src/plugins/user-loader.ts +2 -1
  160. package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
  161. package/src/providers/inference/endpoint-probe.ts +27 -12
  162. package/src/providers/openai/responses-provider.ts +9 -2
  163. package/src/providers/opencode/client.test.ts +167 -0
  164. package/src/providers/opencode/client.ts +73 -4
  165. package/src/runtime/AGENTS.md +4 -0
  166. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
  167. package/src/runtime/http-server.ts +6 -6
  168. package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
  169. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +74 -4
  170. package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
  171. package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
  172. package/src/runtime/routes/app-management-routes.ts +17 -2
  173. package/src/runtime/routes/browser-context.ts +39 -0
  174. package/src/runtime/routes/browser-routes.ts +16 -47
  175. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  176. package/src/runtime/routes/credential-routes.ts +1 -4
  177. package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
  178. package/src/runtime/routes/desktop-setup-routes.ts +7 -5
  179. package/src/runtime/routes/mcp-auth-routes.ts +106 -94
  180. package/src/runtime/routes/oauth-commands-routes.ts +14 -0
  181. package/src/runtime/routes/oauth-providers.ts +7 -0
  182. package/src/runtime/routes/plugins-routes.ts +14 -1
  183. package/src/runtime/sync/resource-sync-events.ts +4 -0
  184. package/src/tools/browser/browser-execution.ts +44 -13
  185. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  186. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  187. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  188. package/src/tools/computer-use/definitions.ts +3 -4
  189. package/src/tools/ui-surface/channel-variants.ts +10 -59
  190. package/src/util/abort-reasons.ts +12 -1
  191. package/src/util/plugin-manifest.ts +203 -0
  192. package/src/watch/watch-retro.ts +6 -8
  193. package/src/desktop/desktop-feature.test.ts +0 -18
  194. package/src/desktop/desktop-feature.ts +0 -19
@@ -43,6 +43,7 @@ import type { EditEngineResult } from "../tools/shared/filesystem/edit-engine.js
43
43
  import { applyEdit } from "../tools/shared/filesystem/edit-engine.js";
44
44
  import { getLogger } from "../util/logger.js";
45
45
  import { getDataDir, getWorkspacePluginsDir } from "../util/platform.js";
46
+ import { hasPluginManifest } from "../util/plugin-manifest.js";
46
47
 
47
48
  const log = getLogger("app-store");
48
49
 
@@ -787,8 +788,8 @@ function listAppsForPlugin(
787
788
  * `<workspace>/plugins/<name>/apps/`.
788
789
  *
789
790
  * Plugin discovery mirrors the plugin loader's `scanPlugins`: a plugin is an
790
- * entry that resolves to a directory (following symlinks) and carries a
791
- * `package.json` manifest. Stray directories without a manifest are ignored,
791
+ * entry that resolves to a directory (following symlinks) and carries a root
792
+ * plugin manifest. Stray directories without a manifest are ignored,
792
793
  * and disabled plugins (those with a `.disabled` sentinel) contribute nothing,
793
794
  * matching how their other surfaces (tools, hooks, routes) are gated.
794
795
  */
@@ -813,7 +814,7 @@ export function listPluginApps(): EnumeratedApp[] {
813
814
  } catch {
814
815
  continue;
815
816
  }
816
- if (!existsSync(join(pluginDir, "package.json"))) {
817
+ if (!hasPluginManifest(pluginDir)) {
817
818
  continue;
818
819
  }
819
820
  if (isPluginDisabled(name)) {
@@ -874,7 +875,7 @@ function isSafeIdSegment(segment: string): boolean {
874
875
  * (`plugins~<name>~<app>`, resolved by direct path build). Returns null when
875
876
  * the app does not exist, when the id is not a safe path segment, or when a
876
877
  * plugin id fails the same installed-plugin gates as discovery (directory,
877
- * `package.json` manifest, not disabled).
878
+ * root manifest, not disabled).
878
879
  */
879
880
  export function resolveAppSource(id: string): ResolvedAppSource | null {
880
881
  if (id.startsWith(PLUGIN_APP_ID_PREFIX)) {
@@ -900,7 +901,7 @@ export function resolveAppSource(id: string): ResolvedAppSource | null {
900
901
  } catch {
901
902
  return null;
902
903
  }
903
- if (!existsSync(join(pluginDir, "package.json"))) {
904
+ if (!hasPluginManifest(pluginDir)) {
904
905
  return null;
905
906
  }
906
907
  if (isPluginDisabled(pluginName)) {
@@ -32,7 +32,8 @@ import {
32
32
  } from "../tools/browser/browser-execution.js";
33
33
  import { browserManager } from "../tools/browser/browser-manager.js";
34
34
  import { normalizeBrowserMode } from "../tools/browser/browser-mode.js";
35
- import type { ToolContext, ToolExecutionResult } from "../tools/types.js";
35
+ import type { ToolExecutionResult } from "../tools/types.js";
36
+ import type { BrowserOperationContext as ToolContext } from "./types.js";
36
37
  import type { BrowserOperation } from "./types.js";
37
38
 
38
39
  // ── Dispatch handlers ────────────────────────────────────────────────
@@ -1,3 +1,10 @@
1
+ import type { ScopedCdpClient } from "../tools/browser/cdp-client/types.js";
2
+ import type { ToolContext } from "../tools/types.js";
3
+
4
+ export interface BrowserOperationContext extends ToolContext {
5
+ cdpClient?: ScopedCdpClient;
6
+ }
7
+
1
8
  /**
2
9
  * Canonical browser operation identifiers and typed metadata.
3
10
  *
@@ -0,0 +1,34 @@
1
+ import { getConfig } from "../config/loader.js";
2
+ import { isVirtualDesktopEnabled } from "../desktop/virtual-desktop-feature.js";
3
+ import { browserManager } from "../tools/browser/browser-manager.js";
4
+ import { normalizeBrowserMode } from "../tools/browser/browser-mode.js";
5
+ import { getPinnedTab } from "../tools/browser/pinned-tabs.js";
6
+ import type { ToolContext } from "../tools/types.js";
7
+
8
+ export function shouldUseVirtualDesktopBrowser(
9
+ desktop: boolean | undefined,
10
+ input: Record<string, unknown>,
11
+ context: ToolContext,
12
+ ): boolean {
13
+ if (desktop !== undefined) {
14
+ return desktop;
15
+ }
16
+ const mode = normalizeBrowserMode(input.browser_mode);
17
+ if (
18
+ "error" in mode ||
19
+ mode.mode !== "auto" ||
20
+ input.target_client_id ||
21
+ input.use_active_tab ||
22
+ context.transportInterface !== "web" ||
23
+ context.clientOs === "macos" ||
24
+ context.clientOs === "windows" ||
25
+ context.clientOs === "linux" ||
26
+ context.trustClass !== "guardian" ||
27
+ !context.sourceActorPrincipalId ||
28
+ browserManager.getPreferredBackendKind(context.conversationId) !== null ||
29
+ getPinnedTab(context.conversationId)
30
+ ) {
31
+ return false;
32
+ }
33
+ return isVirtualDesktopEnabled(getConfig());
34
+ }
@@ -185,6 +185,29 @@ describe("createVoiceProgressNarrator", () => {
185
185
  expect(Date.now() - startedAt).toBeLessThan(1000);
186
186
  });
187
187
 
188
+ test("an update that runs out of budget aborts as a narration timeout, not a session abort", async () => {
189
+ // The budget lapsing means this beat goes unspoken; it does not mean the
190
+ // call ended. Sharing `voice_session_aborted` made a narrator whose budget
191
+ // sat below the model's roundtrip read in the logs as a session dying once
192
+ // per update, which is what hid a turn that had gone silent for minutes.
193
+ let seen: AbortSignal | undefined;
194
+ const narrator = createVoiceProgressNarrator({
195
+ config: VoiceProgressConfigSchema.parse({ generationTimeoutMs: 20 }),
196
+ getProvider: async () =>
197
+ stubProvider((_messages, options) => {
198
+ seen = options?.signal;
199
+ return new Promise<ProviderResponse>(() => {});
200
+ }),
201
+ });
202
+
203
+ expect(await narrator.generateProgressText(progressInput)).toBeNull();
204
+ expect(seen?.aborted).toBe(true);
205
+ expect(seen?.reason).toMatchObject({
206
+ kind: "voice_progress_narration_timeout",
207
+ source: "voice-progress-narration",
208
+ });
209
+ });
210
+
188
211
  test("a caller abort settles promptly", async () => {
189
212
  const narrator = createVoiceProgressNarrator({
190
213
  config: VoiceProgressConfigSchema.parse({
@@ -6,7 +6,10 @@ import {
6
6
  HOLD_VERDICT_TOKEN,
7
7
  isIncompleteControlMarkerTail,
8
8
  MINIMIZE_ROOM_MARKER,
9
+ parseTerminalSessionControl,
10
+ type SessionControlRequest,
9
11
  stripInternalSpeechMarkers,
12
+ terminalControlMarkerLength,
10
13
  } from "../voice-control-protocol.js";
11
14
 
12
15
  describe("front-door verdict tokens", () => {
@@ -136,3 +139,69 @@ describe("createControlMarkerHoldback", () => {
136
139
  expect(chunks).toEqual([]);
137
140
  });
138
141
  });
142
+
143
+ describe("session control markers", () => {
144
+ test("strips untimed and timed mute markers", () => {
145
+ expect(stripInternalSpeechMarkers("Muted. [MUTE]").trim()).toBe("Muted.");
146
+ expect(stripInternalSpeechMarkers("Muted. [MUTE:30]").trim()).toBe(
147
+ "Muted.",
148
+ );
149
+ });
150
+
151
+ test("holds a streaming timed mute until its bracket arrives", () => {
152
+ expect(isIncompleteControlMarkerTail("[MU")).toBe(true);
153
+ expect(isIncompleteControlMarkerTail("[MUTE:3")).toBe(true);
154
+ expect(isIncompleteControlMarkerTail("[MUTE:30]")).toBe(false);
155
+ expect(isIncompleteControlMarkerTail("[UPDATES:FEW")).toBe(true);
156
+ expect(isIncompleteControlMarkerTail("[UPDATES:FEWER]")).toBe(false);
157
+ expect(isIncompleteControlMarkerTail("[LOOK:SCR")).toBe(true);
158
+ expect(
159
+ stripInternalSpeechMarkers("Taking a look. [LOOK:SCREEN]").trim(),
160
+ ).toBe("Taking a look.");
161
+ });
162
+
163
+ test.each([
164
+ ["Okay, talk soon. [END_CALL]", { action: "end" }],
165
+ ["Muted. [MUTE]", { action: "mute" }],
166
+ ["Taking a look. [LOOK:SCREEN]", { action: "look_screen" }],
167
+ ["Show me. [LOOK:CAMERA]", { action: "look_camera" }],
168
+ ["Okay, I'll stop looking. [LOOK:STOP]", { action: "look_stop" }],
169
+ [
170
+ "I'll check in less. [UPDATES:FEWER]",
171
+ { action: "updates", cadence: "fewer" },
172
+ ],
173
+ [
174
+ "Updates are back on. [UPDATES:NORMAL]",
175
+ { action: "updates", cadence: "normal" },
176
+ ],
177
+ [
178
+ "Muting for half a minute. [MUTE:30] ",
179
+ { action: "mute", durationMs: 30_000 },
180
+ ],
181
+ ["Muted. [MUTE: 1.5]", { action: "mute", durationMs: 1_500 }],
182
+ // A garbled or oversized duration still mutes, just until unmuted.
183
+ ["Muted. [MUTE:soon]", { action: "mute" }],
184
+ ["Muted. [MUTE:0]", { action: "mute" }],
185
+ ["Muted. [MUTE:99999]", { action: "mute" }],
186
+ ] as Array<[string, SessionControlRequest]>)(
187
+ "parses the terminal control in %j",
188
+ (text, expected) => {
189
+ expect(parseTerminalSessionControl(text)).toEqual(expected);
190
+ },
191
+ );
192
+
193
+ test("a marker anywhere but the end controls nothing", () => {
194
+ expect(
195
+ parseTerminalSessionControl("Say [END_CALL] and I would hang up."),
196
+ ).toBeNull();
197
+ expect(parseTerminalSessionControl("No markers here.")).toBeNull();
198
+ });
199
+
200
+ test("measures the terminal marker the transcript pass strips", () => {
201
+ expect(terminalControlMarkerLength("Done [-1]")).toBe(4);
202
+ expect(terminalControlMarkerLength("Bye [END_CALL] ")).toBe(10);
203
+ expect(terminalControlMarkerLength("Muted [MUTE:30]")).toBe(9);
204
+ expect(terminalControlMarkerLength("Okay [UPDATES:FEWER]")).toBe(15);
205
+ expect(terminalControlMarkerLength("The array [-1] sorts")).toBe(0);
206
+ });
207
+ });
@@ -2146,6 +2146,68 @@ describe("front-door leg tool suppression", () => {
2146
2146
  });
2147
2147
  });
2148
2148
 
2149
+ describe("desktop skill preactivation", () => {
2150
+ async function preactivatedFor(turn: Record<string, unknown>): Promise<{
2151
+ skillIds: string[];
2152
+ proxyInterfaces: unknown[];
2153
+ skillIdsDuringLoop: string[];
2154
+ }> {
2155
+ const skillIds: string[] = [];
2156
+ const proxyInterfaces: unknown[] = [];
2157
+ let skillIdsDuringLoop: string[] = [];
2158
+ const fake = makeFakeConversation({
2159
+ processing: false,
2160
+ runAgentLoop: async () => {
2161
+ skillIdsDuringLoop = [...skillIds];
2162
+ },
2163
+ });
2164
+ Object.assign(fake.conversation, {
2165
+ addPreactivatedSkillId: (id: string) => {
2166
+ skillIds.push(id);
2167
+ },
2168
+ ensureHostProxiesForTurn: (sourceInterface: unknown) => {
2169
+ proxyInterfaces.push(sourceInterface);
2170
+ },
2171
+ });
2172
+ fakeConversation = fake.conversation;
2173
+
2174
+ await startVoiceTurn({
2175
+ ...makeTurnOptions(undefined, "conv-desktop-skills"),
2176
+ userMessageInterface: "macos",
2177
+ ...turn,
2178
+ });
2179
+ await flushMicrotasks();
2180
+ return { skillIds, proxyInterfaces, skillIdsDuringLoop };
2181
+ }
2182
+
2183
+ test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
2184
+ const result = await preactivatedFor({
2185
+ routingLeg: "escalated",
2186
+ macosDesktopSession: true,
2187
+ });
2188
+
2189
+ expect(result.skillIdsDuringLoop).toContain("computer-use");
2190
+ expect(result.proxyInterfaces).toEqual(["macos"]);
2191
+ });
2192
+
2193
+ test("a session from any other client leaves the desktop skills to be loaded", async () => {
2194
+ const result = await preactivatedFor({ routingLeg: "escalated" });
2195
+
2196
+ expect(result.skillIds).toEqual([]);
2197
+ expect(result.proxyInterfaces).toEqual([]);
2198
+ });
2199
+
2200
+ test("the toolless front-door leg is never preactivated", async () => {
2201
+ const result = await preactivatedFor({
2202
+ routingLeg: "front-door",
2203
+ macosDesktopSession: true,
2204
+ });
2205
+
2206
+ expect(result.skillIds).toEqual([]);
2207
+ expect(result.proxyInterfaces).toEqual([]);
2208
+ });
2209
+ });
2210
+
2149
2211
  describe("cutFrontDoorContentAtVerdict", () => {
2150
2212
  test("null when the content carries no verdict token (committed answer)", () => {
2151
2213
  expect(
@@ -2513,6 +2575,25 @@ describe("transcript hygiene (teardown pass)", () => {
2513
2575
  expect(events).toContain("loadFromDb");
2514
2576
  });
2515
2577
 
2578
+ test("a row ending with a session control marker persists with it stripped", async () => {
2579
+ const { events } = makeReservedRowConversation();
2580
+ getMessageByIdImpl = () =>
2581
+ makeRow("Muting you for thirty seconds. [MUTE:30]");
2582
+
2583
+ await startVoiceTurn({ ...makeTurnOptions(), routingLeg: "front-door" });
2584
+ await flushMicrotasks();
2585
+
2586
+ expect(crudLog.updates).toEqual([
2587
+ {
2588
+ messageId: "assistant-row-1",
2589
+ content: JSON.stringify([
2590
+ { type: "text", text: "Muting you for thirty seconds." },
2591
+ ]),
2592
+ },
2593
+ ]);
2594
+ expect(events).toContain("loadFromDb");
2595
+ });
2596
+
2516
2597
  test("a mid-text [-1] (content, not command) leaves the row untouched", async () => {
2517
2598
  const { events } = makeReservedRowConversation();
2518
2599
  getMessageByIdImpl = () =>
@@ -16,7 +16,7 @@ import type {
16
16
  ProviderResponse,
17
17
  ToolDefinition,
18
18
  } from "../providers/types.js";
19
- import { createAbortReason } from "../util/abort-reasons.js";
19
+ import { createAbortReason, isAbortReason } from "../util/abort-reasons.js";
20
20
  import { getLogger } from "../util/logger.js";
21
21
 
22
22
  const log = getLogger("voice-progress-narration");
@@ -164,7 +164,10 @@ async function requestBoundedResponse(args: {
164
164
  const timeoutTimer = setTimeout(
165
165
  () =>
166
166
  timeoutController.abort(
167
- createAbortReason("voice_session_aborted", "voice-progress-narration"),
167
+ createAbortReason(
168
+ "voice_progress_narration_timeout",
169
+ "voice-progress-narration",
170
+ ),
168
171
  ),
169
172
  args.timeoutMs,
170
173
  );
@@ -260,9 +263,19 @@ export function createVoiceProgressNarrator(options: {
260
263
  }
261
264
  return trimmed;
262
265
  } catch (error) {
266
+ // Why the beat went unspoken, as one greppable field. A narrator whose
267
+ // budget is below the model's real roundtrip drops every update at the
268
+ // deadline and reads in the logs as a run of unrelated failures; naming
269
+ // the outcome is what makes that run legible as "the feature is off".
270
+ const outcome = isAbortReason(error)
271
+ ? error.kind === "voice_progress_narration_timeout"
272
+ ? "timed_out"
273
+ : "session_ended"
274
+ : "error";
263
275
  log.info(
264
276
  {
265
277
  error,
278
+ outcome,
266
279
  providerResolveMs,
267
280
  totalMs: Math.round(performance.now() - startedAt),
268
281
  timeoutMs: options.config.generationTimeoutMs,
@@ -37,6 +37,41 @@ export const ESCALATE_VERDICT_TOKEN = "[1]";
37
37
  */
38
38
  export const MINIMIZE_ROOM_MARKER = "[-1]";
39
39
 
40
+ /**
41
+ * Mic-mute session control for live voice: `[MUTE]` mutes until the user
42
+ * unmutes, `[MUTE:<seconds>]` mutes for that long. Like {@link END_CALL_MARKER}
43
+ * on a live-voice session, it is acted on only at the very end of a reply
44
+ * (see {@link parseTerminalSessionControl}), after the acknowledgement that
45
+ * precedes it has been spoken.
46
+ */
47
+ export const MUTE_MARKER = "[MUTE]";
48
+ const MUTE_MARKER_PREFIX = "[MUTE:";
49
+
50
+ /** Longest timed mute a marker can ask for; longer asks mute until unmuted. */
51
+ export const MAX_TIMED_MUTE_SECONDS = 3600;
52
+
53
+ /**
54
+ * Progress-update cadence for the rest of a live-voice session:
55
+ * {@link FEWER_UPDATES_MARKER} after "don't give me updates so often",
56
+ * {@link NORMAL_UPDATES_MARKER} to go back. Terminal like the other session
57
+ * controls, but carried out by the session itself: narration is the daemon's.
58
+ */
59
+ export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
60
+ export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
61
+ const UPDATES_MARKER_PREFIX = "[UPDATES:";
62
+
63
+ /**
64
+ * Look session controls for live voice: turn on a screen share
65
+ * ({@link LOOK_SCREEN_MARKER}) or the camera ({@link LOOK_CAMERA_MARKER}) so
66
+ * the call can see what the user means by "take a look". Carried out by the
67
+ * client, like mute and end.
68
+ */
69
+ export const LOOK_SCREEN_MARKER = "[LOOK:SCREEN]";
70
+ export const LOOK_CAMERA_MARKER = "[LOOK:CAMERA]";
71
+ /** Stop showing the call the screen and the camera, whichever is on. */
72
+ export const LOOK_STOP_MARKER = "[LOOK:STOP]";
73
+ const LOOK_MARKER_PREFIX = "[LOOK:";
74
+
40
75
  // ---------------------------------------------------------------------------
41
76
  // Regexes
42
77
  // ---------------------------------------------------------------------------
@@ -57,6 +92,9 @@ const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
57
92
  const HOLD_VERDICT_TOKEN_REGEX = /\[0\]/g;
58
93
  const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
59
94
  const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
95
+ const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
96
+ const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
97
+ const LOOK_MARKER_REGEX = /\[LOOK:\s*[^\]]*\]/g;
60
98
  const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
61
99
  const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
62
100
 
@@ -182,6 +220,9 @@ export function stripInternalSpeechMarkers(text: string): string {
182
220
  .replace(HOLD_VERDICT_TOKEN_REGEX, "")
183
221
  .replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
184
222
  .replace(MINIMIZE_ROOM_MARKER_REGEX, "")
223
+ .replace(MUTE_MARKER_REGEX, "")
224
+ .replace(UPDATES_MARKER_REGEX, "")
225
+ .replace(LOOK_MARKER_REGEX, "")
185
226
  .replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
186
227
  .replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
187
228
  return result;
@@ -207,6 +248,10 @@ const CONTROL_MARKER_STRINGS = [
207
248
  "[0]",
208
249
  "[1]",
209
250
  "[-1]",
251
+ MUTE_MARKER,
252
+ MUTE_MARKER_PREFIX,
253
+ UPDATES_MARKER_PREFIX,
254
+ LOOK_MARKER_PREFIX,
210
255
  "[GUARDIAN_TIMEOUT]",
211
256
  "[GUARDIAN_UNAVAILABLE]",
212
257
  ];
@@ -220,6 +265,9 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
220
265
  "[ASK_GUARDIAN:",
221
266
  "[USER_ANSWERED:",
222
267
  "[USER_INSTRUCTION:",
268
+ MUTE_MARKER_PREFIX,
269
+ UPDATES_MARKER_PREFIX,
270
+ LOOK_MARKER_PREFIX,
223
271
  ];
224
272
 
225
273
  const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
@@ -299,3 +347,87 @@ export function createControlMarkerHoldback(
299
347
  }
300
348
  };
301
349
  }
350
+
351
+ // ---------------------------------------------------------------------------
352
+ // Live-voice session controls
353
+ // ---------------------------------------------------------------------------
354
+
355
+ /**
356
+ * A session control a live-voice reply asked for with a terminal marker:
357
+ * `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
358
+ * timed form, `updates` from the progress-cadence markers, `look_screen` and
359
+ * `look_camera` and `look_stop` from the look markers.
360
+ */
361
+ export type SessionControlRequest =
362
+ | { readonly action: "end" }
363
+ | { readonly action: "mute"; readonly durationMs?: number }
364
+ | { readonly action: "updates"; readonly cadence: "fewer" | "normal" }
365
+ | { readonly action: "look_screen" }
366
+ | { readonly action: "look_camera" }
367
+ | { readonly action: "look_stop" };
368
+
369
+ const TERMINAL_SESSION_CONTROL_REGEX =
370
+ /(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[LOOK:(SCREEN|CAMERA|STOP)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
371
+
372
+ /**
373
+ * The session control a reply ends with, or null.
374
+ *
375
+ * **Terminal position only.** The marker follows the spoken acknowledgement
376
+ * ("Okay, talk soon."), so a reply that merely mentions a marker mid-text, or
377
+ * parrots one from history before carrying on, controls nothing. This is the
378
+ * same rule the minimize marker's transcript hygiene applies.
379
+ *
380
+ * A timed mute whose body is not a positive number of seconds degrades to an
381
+ * untimed mute rather than to nothing: the user asked to be muted, and muting
382
+ * until they unmute is the conservative reading of a garbled duration. Asks
383
+ * past {@link MAX_TIMED_MUTE_SECONDS} degrade the same way.
384
+ */
385
+ export function parseTerminalSessionControl(
386
+ text: string,
387
+ ): SessionControlRequest | null {
388
+ const match = TERMINAL_SESSION_CONTROL_REGEX.exec(text);
389
+ if (!match) {
390
+ return null;
391
+ }
392
+ if (match[1] === END_CALL_MARKER) {
393
+ return { action: "end" };
394
+ }
395
+ if (match[2] !== undefined) {
396
+ return {
397
+ action: "updates",
398
+ cadence: match[2] === "FEWER" ? "fewer" : "normal",
399
+ };
400
+ }
401
+ if (match[3] !== undefined) {
402
+ return {
403
+ action:
404
+ match[3] === "SCREEN"
405
+ ? "look_screen"
406
+ : match[3] === "CAMERA"
407
+ ? "look_camera"
408
+ : "look_stop",
409
+ };
410
+ }
411
+ const seconds = match[4] === undefined ? NaN : Number(match[4].trim());
412
+ if (
413
+ Number.isFinite(seconds) &&
414
+ seconds > 0 &&
415
+ seconds <= MAX_TIMED_MUTE_SECONDS
416
+ ) {
417
+ return { action: "mute", durationMs: Math.round(seconds * 1000) };
418
+ }
419
+ return { action: "mute" };
420
+ }
421
+
422
+ /**
423
+ * Length of the marker a row's text ends with, ignoring trailing whitespace,
424
+ * when it is one the transcript hygiene pass strips from the persisted row
425
+ * (the minimize marker or a session control); 0 otherwise.
426
+ */
427
+ export function terminalControlMarkerLength(text: string): number {
428
+ const trimmed = text.trimEnd();
429
+ if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
430
+ return MINIMIZE_ROOM_MARKER.length;
431
+ }
432
+ return TERMINAL_SESSION_CONTROL_REGEX.exec(trimmed)?.[1]?.length ?? 0;
433
+ }
@@ -31,6 +31,7 @@ import {
31
31
  import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
32
32
  import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
33
33
  import { getOrCreateConversation } from "../daemon/conversation-store.js";
34
+ import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
34
35
  import type { TrustContext } from "../daemon/trust-context-types.js";
35
36
  import {
36
37
  newestPersistedSightFrame,
@@ -70,8 +71,8 @@ import {
70
71
  CALL_VERIFICATION_COMPLETE_MARKER,
71
72
  ESCALATE_VERDICT_TOKEN,
72
73
  HOLD_VERDICT_TOKEN,
73
- MINIMIZE_ROOM_MARKER,
74
74
  stripInternalSpeechMarkers,
75
+ terminalControlMarkerLength,
75
76
  } from "./voice-control-protocol.js";
76
77
  import {
77
78
  createFrontDoorStreamGate,
@@ -464,6 +465,14 @@ export interface VoiceTurnOptions {
464
465
  * path has always used.
465
466
  */
466
467
  actorFallbackSuppressed?: boolean;
468
+ /**
469
+ * Whether the session was opened from the macOS desktop client. A leg that
470
+ * can run tools then starts with the desktop skills (computer use, app
471
+ * control, screen annotation) already active, as a typed turn from that
472
+ * client does. The turn interface cannot answer this: every live-voice
473
+ * session reports `macos` for its channel capabilities, iOS included.
474
+ */
475
+ macosDesktopSession?: boolean;
467
476
  /** Whether this is an inbound call (no outbound task). */
468
477
  isInbound: boolean;
469
478
  /** The outbound call task, if any. */
@@ -767,17 +776,22 @@ function stripMarkersFromBlocks(blocks: ContentBlock[]): ContentBlock[] {
767
776
  }
768
777
 
769
778
  /**
770
- * Remove the terminal MINIMIZE_ROOM_MARKER from the end of a row's text,
771
- * walking text blocks from the last one backward so a marker split across
772
- * block boundaries (e.g. `"Done [-"` + `"1]"`) is removed whole — the
773
- * per-block strip in {@link stripMarkersFromBlocks} only sees fragments and
774
- * would leave both halves in place. Callers must have established that the
775
- * row's joined text ends with the marker after trimming trailing whitespace.
779
+ * Remove a terminal control marker (the minimize marker or a session control,
780
+ * `markerLength` characters long) from the end of a row's text, walking text
781
+ * blocks from the last one backward so a marker split across block boundaries
782
+ * (e.g. `"Done [-"` + `"1]"`) is removed whole — the per-block strip in
783
+ * {@link stripMarkersFromBlocks} only sees fragments and would leave both
784
+ * halves in place. Callers must have established, with
785
+ * {@link terminalControlMarkerLength}, that the row's joined text ends with
786
+ * the marker after trimming trailing whitespace.
776
787
  */
777
- function stripTerminalMinimizeMarker(blocks: ContentBlock[]): ContentBlock[] {
788
+ function stripTerminalControlMarker(
789
+ blocks: ContentBlock[],
790
+ markerLength: number,
791
+ ): ContentBlock[] {
778
792
  const result = blocks.map((block) => ({ ...block }));
779
793
  const joined = joinedTextOfBlocks(result);
780
- const cutAt = joined.trimEnd().length - MINIMIZE_ROOM_MARKER.length;
794
+ const cutAt = joined.trimEnd().length - markerLength;
781
795
  let blockEnd = joined.length;
782
796
  for (let i = result.length - 1; i >= 0 && blockEnd > cutAt; i--) {
783
797
  const block = result[i]!;
@@ -1810,14 +1824,14 @@ export async function startVoiceTurn(
1810
1824
  * never the verdict token or the text streamed past the cap (issue
1811
1825
  * #37850). A row with no spoken bridge (canned-fallback case — that
1812
1826
  * bridge is audio-only) is deleted.
1813
- * - Any leg whose row ENDS with the `[-1]` minimize marker (swallowed
1814
- * before TTS on the live path) has its text blocks rewritten through
1827
+ * - Any leg whose row ENDS with the `[-1]` minimize marker or a session
1828
+ * control marker (`[END_CALL]`, `[MUTE]`, `[MUTE:<seconds>]`), all
1829
+ * swallowed before TTS, has its text blocks rewritten through
1815
1830
  * `stripInternalSpeechMarkers` so the marker never renders in the chat
1816
- * transcript. This covers front-door answers too: that leg is never
1817
- * taught the marker, but it can parrot one from visible conversation
1818
- * history, and the parroted marker is never spoken and never minimizes
1819
- * the room. Deliberately scoped to that marker: rows without it
1820
- * persist byte-identical.
1831
+ * transcript. This covers front-door answers too: a front-door answer
1832
+ * may end with a session control, and it can parrot `[-1]` from visible
1833
+ * conversation history. Deliberately scoped to terminal markers: rows
1834
+ * without one persist byte-identical.
1821
1835
  *
1822
1836
  * After a rewrite, in-memory history is reloaded from the clean DB before
1823
1837
  * the escalated leg — blocked on this turn's teardown — snapshots it, so
@@ -1851,6 +1865,9 @@ export async function startVoiceTurn(
1851
1865
  action = "delete_discarded";
1852
1866
  } else {
1853
1867
  const row = getMessageById(reservedAssistantRowId, opts.conversationId);
1868
+ const terminalMarkerLength = row
1869
+ ? terminalControlMarkerLength(joinedTextOfBlocks(row.content))
1870
+ : 0;
1854
1871
  const cut =
1855
1872
  row && opts.routingLeg === "front-door"
1856
1873
  ? cutFrontDoorContentAtVerdict(row.content)
@@ -1869,19 +1886,18 @@ export async function startVoiceTurn(
1869
1886
  action = "delete_empty";
1870
1887
  }
1871
1888
  } else if (
1872
- // Terminal position only — mirrors the live latch in
1873
- // createControlMarkerHoldback: a reply whose CONTENT contains
1874
- // "[-1]" mid-text never minimized the room, so its transcript
1875
- // keeps that content untouched too. Front-door answer rows (no
1876
- // verdict token to cut) take this branch as well.
1877
- joinedTextOfBlocks(row.content)
1878
- .trimEnd()
1879
- .endsWith(MINIMIZE_ROOM_MARKER)
1889
+ // Terminal position only — mirrors parseTerminalSessionControl: a
1890
+ // reply whose CONTENT contains a marker mid-text never acted on it,
1891
+ // so its transcript keeps that content untouched too. Front-door
1892
+ // answer rows (no verdict token to cut) take this branch as well.
1893
+ terminalMarkerLength > 0
1880
1894
  ) {
1881
1895
  // Terminal marker first (boundary-aware — it may span text blocks),
1882
1896
  // then the per-block strip for any interior complete markers.
1883
1897
  const cleaned = trimOuterTextEdges(
1884
- stripMarkersFromBlocks(stripTerminalMinimizeMarker(row.content)),
1898
+ stripMarkersFromBlocks(
1899
+ stripTerminalControlMarker(row.content, terminalMarkerLength),
1900
+ ),
1885
1901
  );
1886
1902
  // A marker-only reply (the model said nothing beyond "[-1]") strips
1887
1903
  // to nothing at all; keeping the row would render a blank assistant
@@ -1895,7 +1911,7 @@ export async function startVoiceTurn(
1895
1911
  reservedAssistantRowId,
1896
1912
  JSON.stringify(cleaned),
1897
1913
  );
1898
- action = "strip_minimize_marker";
1914
+ action = "strip_control_marker";
1899
1915
  }
1900
1916
  }
1901
1917
  }
@@ -2009,6 +2025,20 @@ export async function startVoiceTurn(
2009
2025
  (needsImagePin
2010
2026
  ? VOICE_IMAGE_PROFILE
2011
2027
  : (conversationProfile?.profile ?? null));
2028
+ if (opts.macosDesktopSession === true && !frontDoorToolsSuppressed) {
2029
+ const sourceInterface = turnInterfaceContext.userMessageInterface;
2030
+ const sourceActorPrincipalId =
2031
+ voiceTurnValues.actorPrincipalId ?? undefined;
2032
+ conversation.ensureHostProxiesForTurn(
2033
+ sourceInterface,
2034
+ sourceActorPrincipalId,
2035
+ );
2036
+ preactivateHostProxySkills(
2037
+ conversation,
2038
+ sourceInterface,
2039
+ sourceActorPrincipalId,
2040
+ );
2041
+ }
2012
2042
  await conversation.runAgentLoop(persistedContent, messageId, {
2013
2043
  onEvent: (msg: AssistantEvent) => {
2014
2044
  if (msg.type === "assistant_turn_start") {