@vellumai/assistant 0.12.2-staging.1 → 0.12.2-staging.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/desktop-browser-cli.md +33 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
- package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
- package/openapi.yaml +50 -4
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +279 -0
- package/src/__tests__/always-loaded-tools-guard.test.ts +5 -5
- package/src/__tests__/config-schema.test.ts +1 -1
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
- package/src/__tests__/credential-routes.test.ts +38 -0
- package/src/__tests__/credential-security-invariants.test.ts +1 -1
- package/src/__tests__/cu-unified-flow.test.ts +6 -2
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/host-cu-proxy.test.ts +185 -30
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/mcp-auth-routes.test.ts +99 -0
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-health-check.test.ts +32 -32
- package/src/__tests__/mcp-list-plugin-servers.test.ts +77 -32
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/oauth-commands-routes.test.ts +55 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +22 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
- package/src/__tests__/skills.test.ts +10 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +51 -0
- package/src/__tests__/ui-channel-variants.test.ts +1 -56
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
- package/src/acp/__tests__/acp-credentials.test.ts +13 -0
- package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
- package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
- package/src/acp/acp-claude-oauth.ts +328 -14
- package/src/acp/acp-credentials.ts +19 -0
- package/src/acp/claude-token-refresh.ts +150 -0
- package/src/acp/prepare-agent-env.ts +21 -6
- package/src/apps/app-store.ts +6 -5
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/calls/__tests__/progress-narration.test.ts +23 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +69 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +81 -0
- package/src/calls/progress-narration.ts +15 -2
- package/src/calls/voice-control-protocol.ts +132 -0
- package/src/calls/voice-session-bridge.ts +56 -26
- package/src/calls/voice-triage-escalate.ts +3 -3
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/oauth/status.ts +5 -0
- package/src/cli/commands/plugins.ts +16 -5
- package/src/cli/lib/__tests__/install-from-github.test.ts +111 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/plugin-details.test.ts +26 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +13 -0
- package/src/cli/lib/install-from-github.ts +17 -18
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-details.ts +21 -8
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/bundled-skills/computer-use/SKILL.md +43 -4
- package/src/config/bundled-skills/computer-use/TOOLS.json +3 -3
- package/src/config/bundled-skills/screen-annotation/SKILL.md +2 -2
- package/src/config/feature-flag-registry.json +11 -3
- package/src/config/loader.ts +1 -0
- package/src/config/schemas/__tests__/voice.test.ts +2 -2
- package/src/config/schemas/mcp.ts +9 -7
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/services.ts +10 -0
- package/src/config/schemas/voice.ts +2 -2
- package/src/config/skills.ts +13 -21
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +1 -0
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-runtime-assembly.ts +19 -7
- package/src/daemon/conversation-surfaces.ts +6 -1
- package/src/daemon/conversation-tool-setup.ts +37 -43
- package/src/daemon/host-cu-proxy.ts +94 -31
- package/src/daemon/host-proxy-preactivation.ts +2 -0
- package/src/daemon/mcp-reload-service.ts +7 -0
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/providers-setup.ts +4 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-automation-lease.test.ts +208 -0
- package/src/desktop/desktop-automation-lease.ts +270 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +58 -8
- package/src/desktop/desktop-dependencies.test.ts +29 -0
- package/src/desktop/desktop-dependencies.ts +26 -5
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.ts +14 -1
- package/src/desktop/desktop-session-manager.test.ts +86 -3
- package/src/desktop/desktop-session-manager.ts +144 -24
- package/src/desktop/desktop-stream-bridge.ts +1 -1
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/i18n/__tests__/i18n.test.ts +9 -5
- package/src/i18n/messages.ts +13 -4
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
- package/src/live-voice/__tests__/live-voice-progress.test.ts +79 -0
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/protocol.test.ts +47 -0
- package/src/live-voice/__tests__/session-controls.test.ts +113 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-session.ts +88 -7
- package/src/live-voice/protocol.ts +69 -0
- package/src/live-voice/session-controls.ts +139 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +11 -0
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +42 -15
- package/src/mcp/manager.ts +61 -8
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/oauth/__tests__/seed-providers-managed.test.ts +95 -0
- package/src/oauth/connection-resolver.test.ts +27 -0
- package/src/oauth/connection-resolver.ts +25 -1
- package/src/oauth/provider-serializer.ts +9 -0
- package/src/oauth/seed-providers.ts +121 -2
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +9 -15
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +45 -0
- package/src/providers/inference/endpoint-probe.ts +27 -12
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +167 -0
- package/src/providers/opencode/client.ts +73 -4
- package/src/runtime/AGENTS.md +4 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +45 -2
- package/src/runtime/http-server.ts +6 -6
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
- package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +74 -4
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +25 -3
- package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
- package/src/runtime/routes/app-management-routes.ts +17 -2
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/credential-routes.ts +1 -4
- package/src/runtime/routes/desktop-setup-routes.test.ts +16 -1
- package/src/runtime/routes/desktop-setup-routes.ts +7 -5
- package/src/runtime/routes/mcp-auth-routes.ts +106 -94
- package/src/runtime/routes/oauth-commands-routes.ts +14 -0
- package/src/runtime/routes/oauth-providers.ts +7 -0
- package/src/runtime/routes/plugins-routes.ts +14 -1
- package/src/runtime/sync/resource-sync-events.ts +4 -0
- package/src/tools/browser/browser-execution.ts +44 -13
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/computer-use/definitions.ts +3 -4
- package/src/tools/ui-surface/channel-variants.ts +10 -59
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/plugin-manifest.ts +203 -0
- package/src/watch/watch-retro.ts +6 -8
- package/src/desktop/desktop-feature.test.ts +0 -18
- package/src/desktop/desktop-feature.ts +0 -19
package/src/apps/app-store.ts
CHANGED
|
@@ -43,6 +43,7 @@ import type { EditEngineResult } from "../tools/shared/filesystem/edit-engine.js
|
|
|
43
43
|
import { applyEdit } from "../tools/shared/filesystem/edit-engine.js";
|
|
44
44
|
import { getLogger } from "../util/logger.js";
|
|
45
45
|
import { getDataDir, getWorkspacePluginsDir } from "../util/platform.js";
|
|
46
|
+
import { hasPluginManifest } from "../util/plugin-manifest.js";
|
|
46
47
|
|
|
47
48
|
const log = getLogger("app-store");
|
|
48
49
|
|
|
@@ -787,8 +788,8 @@ function listAppsForPlugin(
|
|
|
787
788
|
* `<workspace>/plugins/<name>/apps/`.
|
|
788
789
|
*
|
|
789
790
|
* Plugin discovery mirrors the plugin loader's `scanPlugins`: a plugin is an
|
|
790
|
-
* entry that resolves to a directory (following symlinks) and carries a
|
|
791
|
-
*
|
|
791
|
+
* entry that resolves to a directory (following symlinks) and carries a root
|
|
792
|
+
* plugin manifest. Stray directories without a manifest are ignored,
|
|
792
793
|
* and disabled plugins (those with a `.disabled` sentinel) contribute nothing,
|
|
793
794
|
* matching how their other surfaces (tools, hooks, routes) are gated.
|
|
794
795
|
*/
|
|
@@ -813,7 +814,7 @@ export function listPluginApps(): EnumeratedApp[] {
|
|
|
813
814
|
} catch {
|
|
814
815
|
continue;
|
|
815
816
|
}
|
|
816
|
-
if (!
|
|
817
|
+
if (!hasPluginManifest(pluginDir)) {
|
|
817
818
|
continue;
|
|
818
819
|
}
|
|
819
820
|
if (isPluginDisabled(name)) {
|
|
@@ -874,7 +875,7 @@ function isSafeIdSegment(segment: string): boolean {
|
|
|
874
875
|
* (`plugins~<name>~<app>`, resolved by direct path build). Returns null when
|
|
875
876
|
* the app does not exist, when the id is not a safe path segment, or when a
|
|
876
877
|
* plugin id fails the same installed-plugin gates as discovery (directory,
|
|
877
|
-
*
|
|
878
|
+
* root manifest, not disabled).
|
|
878
879
|
*/
|
|
879
880
|
export function resolveAppSource(id: string): ResolvedAppSource | null {
|
|
880
881
|
if (id.startsWith(PLUGIN_APP_ID_PREFIX)) {
|
|
@@ -900,7 +901,7 @@ export function resolveAppSource(id: string): ResolvedAppSource | null {
|
|
|
900
901
|
} catch {
|
|
901
902
|
return null;
|
|
902
903
|
}
|
|
903
|
-
if (!
|
|
904
|
+
if (!hasPluginManifest(pluginDir)) {
|
|
904
905
|
return null;
|
|
905
906
|
}
|
|
906
907
|
if (isPluginDisabled(pluginName)) {
|
|
@@ -32,7 +32,8 @@ import {
|
|
|
32
32
|
} from "../tools/browser/browser-execution.js";
|
|
33
33
|
import { browserManager } from "../tools/browser/browser-manager.js";
|
|
34
34
|
import { normalizeBrowserMode } from "../tools/browser/browser-mode.js";
|
|
35
|
-
import type {
|
|
35
|
+
import type { ToolExecutionResult } from "../tools/types.js";
|
|
36
|
+
import type { BrowserOperationContext as ToolContext } from "./types.js";
|
|
36
37
|
import type { BrowserOperation } from "./types.js";
|
|
37
38
|
|
|
38
39
|
// ── Dispatch handlers ────────────────────────────────────────────────
|
package/src/browser/types.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import type { ScopedCdpClient } from "../tools/browser/cdp-client/types.js";
|
|
2
|
+
import type { ToolContext } from "../tools/types.js";
|
|
3
|
+
|
|
4
|
+
export interface BrowserOperationContext extends ToolContext {
|
|
5
|
+
cdpClient?: ScopedCdpClient;
|
|
6
|
+
}
|
|
7
|
+
|
|
1
8
|
/**
|
|
2
9
|
* Canonical browser operation identifiers and typed metadata.
|
|
3
10
|
*
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { getConfig } from "../config/loader.js";
|
|
2
|
+
import { isVirtualDesktopEnabled } from "../desktop/virtual-desktop-feature.js";
|
|
3
|
+
import { browserManager } from "../tools/browser/browser-manager.js";
|
|
4
|
+
import { normalizeBrowserMode } from "../tools/browser/browser-mode.js";
|
|
5
|
+
import { getPinnedTab } from "../tools/browser/pinned-tabs.js";
|
|
6
|
+
import type { ToolContext } from "../tools/types.js";
|
|
7
|
+
|
|
8
|
+
export function shouldUseVirtualDesktopBrowser(
|
|
9
|
+
desktop: boolean | undefined,
|
|
10
|
+
input: Record<string, unknown>,
|
|
11
|
+
context: ToolContext,
|
|
12
|
+
): boolean {
|
|
13
|
+
if (desktop !== undefined) {
|
|
14
|
+
return desktop;
|
|
15
|
+
}
|
|
16
|
+
const mode = normalizeBrowserMode(input.browser_mode);
|
|
17
|
+
if (
|
|
18
|
+
"error" in mode ||
|
|
19
|
+
mode.mode !== "auto" ||
|
|
20
|
+
input.target_client_id ||
|
|
21
|
+
input.use_active_tab ||
|
|
22
|
+
context.transportInterface !== "web" ||
|
|
23
|
+
context.clientOs === "macos" ||
|
|
24
|
+
context.clientOs === "windows" ||
|
|
25
|
+
context.clientOs === "linux" ||
|
|
26
|
+
context.trustClass !== "guardian" ||
|
|
27
|
+
!context.sourceActorPrincipalId ||
|
|
28
|
+
browserManager.getPreferredBackendKind(context.conversationId) !== null ||
|
|
29
|
+
getPinnedTab(context.conversationId)
|
|
30
|
+
) {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
return isVirtualDesktopEnabled(getConfig());
|
|
34
|
+
}
|
|
@@ -185,6 +185,29 @@ describe("createVoiceProgressNarrator", () => {
|
|
|
185
185
|
expect(Date.now() - startedAt).toBeLessThan(1000);
|
|
186
186
|
});
|
|
187
187
|
|
|
188
|
+
test("an update that runs out of budget aborts as a narration timeout, not a session abort", async () => {
|
|
189
|
+
// The budget lapsing means this beat goes unspoken; it does not mean the
|
|
190
|
+
// call ended. Sharing `voice_session_aborted` made a narrator whose budget
|
|
191
|
+
// sat below the model's roundtrip read in the logs as a session dying once
|
|
192
|
+
// per update, which is what hid a turn that had gone silent for minutes.
|
|
193
|
+
let seen: AbortSignal | undefined;
|
|
194
|
+
const narrator = createVoiceProgressNarrator({
|
|
195
|
+
config: VoiceProgressConfigSchema.parse({ generationTimeoutMs: 20 }),
|
|
196
|
+
getProvider: async () =>
|
|
197
|
+
stubProvider((_messages, options) => {
|
|
198
|
+
seen = options?.signal;
|
|
199
|
+
return new Promise<ProviderResponse>(() => {});
|
|
200
|
+
}),
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
expect(await narrator.generateProgressText(progressInput)).toBeNull();
|
|
204
|
+
expect(seen?.aborted).toBe(true);
|
|
205
|
+
expect(seen?.reason).toMatchObject({
|
|
206
|
+
kind: "voice_progress_narration_timeout",
|
|
207
|
+
source: "voice-progress-narration",
|
|
208
|
+
});
|
|
209
|
+
});
|
|
210
|
+
|
|
188
211
|
test("a caller abort settles promptly", async () => {
|
|
189
212
|
const narrator = createVoiceProgressNarrator({
|
|
190
213
|
config: VoiceProgressConfigSchema.parse({
|
|
@@ -6,7 +6,10 @@ import {
|
|
|
6
6
|
HOLD_VERDICT_TOKEN,
|
|
7
7
|
isIncompleteControlMarkerTail,
|
|
8
8
|
MINIMIZE_ROOM_MARKER,
|
|
9
|
+
parseTerminalSessionControl,
|
|
10
|
+
type SessionControlRequest,
|
|
9
11
|
stripInternalSpeechMarkers,
|
|
12
|
+
terminalControlMarkerLength,
|
|
10
13
|
} from "../voice-control-protocol.js";
|
|
11
14
|
|
|
12
15
|
describe("front-door verdict tokens", () => {
|
|
@@ -136,3 +139,69 @@ describe("createControlMarkerHoldback", () => {
|
|
|
136
139
|
expect(chunks).toEqual([]);
|
|
137
140
|
});
|
|
138
141
|
});
|
|
142
|
+
|
|
143
|
+
describe("session control markers", () => {
|
|
144
|
+
test("strips untimed and timed mute markers", () => {
|
|
145
|
+
expect(stripInternalSpeechMarkers("Muted. [MUTE]").trim()).toBe("Muted.");
|
|
146
|
+
expect(stripInternalSpeechMarkers("Muted. [MUTE:30]").trim()).toBe(
|
|
147
|
+
"Muted.",
|
|
148
|
+
);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test("holds a streaming timed mute until its bracket arrives", () => {
|
|
152
|
+
expect(isIncompleteControlMarkerTail("[MU")).toBe(true);
|
|
153
|
+
expect(isIncompleteControlMarkerTail("[MUTE:3")).toBe(true);
|
|
154
|
+
expect(isIncompleteControlMarkerTail("[MUTE:30]")).toBe(false);
|
|
155
|
+
expect(isIncompleteControlMarkerTail("[UPDATES:FEW")).toBe(true);
|
|
156
|
+
expect(isIncompleteControlMarkerTail("[UPDATES:FEWER]")).toBe(false);
|
|
157
|
+
expect(isIncompleteControlMarkerTail("[LOOK:SCR")).toBe(true);
|
|
158
|
+
expect(
|
|
159
|
+
stripInternalSpeechMarkers("Taking a look. [LOOK:SCREEN]").trim(),
|
|
160
|
+
).toBe("Taking a look.");
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test.each([
|
|
164
|
+
["Okay, talk soon. [END_CALL]", { action: "end" }],
|
|
165
|
+
["Muted. [MUTE]", { action: "mute" }],
|
|
166
|
+
["Taking a look. [LOOK:SCREEN]", { action: "look_screen" }],
|
|
167
|
+
["Show me. [LOOK:CAMERA]", { action: "look_camera" }],
|
|
168
|
+
["Okay, I'll stop looking. [LOOK:STOP]", { action: "look_stop" }],
|
|
169
|
+
[
|
|
170
|
+
"I'll check in less. [UPDATES:FEWER]",
|
|
171
|
+
{ action: "updates", cadence: "fewer" },
|
|
172
|
+
],
|
|
173
|
+
[
|
|
174
|
+
"Updates are back on. [UPDATES:NORMAL]",
|
|
175
|
+
{ action: "updates", cadence: "normal" },
|
|
176
|
+
],
|
|
177
|
+
[
|
|
178
|
+
"Muting for half a minute. [MUTE:30] ",
|
|
179
|
+
{ action: "mute", durationMs: 30_000 },
|
|
180
|
+
],
|
|
181
|
+
["Muted. [MUTE: 1.5]", { action: "mute", durationMs: 1_500 }],
|
|
182
|
+
// A garbled or oversized duration still mutes, just until unmuted.
|
|
183
|
+
["Muted. [MUTE:soon]", { action: "mute" }],
|
|
184
|
+
["Muted. [MUTE:0]", { action: "mute" }],
|
|
185
|
+
["Muted. [MUTE:99999]", { action: "mute" }],
|
|
186
|
+
] as Array<[string, SessionControlRequest]>)(
|
|
187
|
+
"parses the terminal control in %j",
|
|
188
|
+
(text, expected) => {
|
|
189
|
+
expect(parseTerminalSessionControl(text)).toEqual(expected);
|
|
190
|
+
},
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
test("a marker anywhere but the end controls nothing", () => {
|
|
194
|
+
expect(
|
|
195
|
+
parseTerminalSessionControl("Say [END_CALL] and I would hang up."),
|
|
196
|
+
).toBeNull();
|
|
197
|
+
expect(parseTerminalSessionControl("No markers here.")).toBeNull();
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("measures the terminal marker the transcript pass strips", () => {
|
|
201
|
+
expect(terminalControlMarkerLength("Done [-1]")).toBe(4);
|
|
202
|
+
expect(terminalControlMarkerLength("Bye [END_CALL] ")).toBe(10);
|
|
203
|
+
expect(terminalControlMarkerLength("Muted [MUTE:30]")).toBe(9);
|
|
204
|
+
expect(terminalControlMarkerLength("Okay [UPDATES:FEWER]")).toBe(15);
|
|
205
|
+
expect(terminalControlMarkerLength("The array [-1] sorts")).toBe(0);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
@@ -2146,6 +2146,68 @@ describe("front-door leg tool suppression", () => {
|
|
|
2146
2146
|
});
|
|
2147
2147
|
});
|
|
2148
2148
|
|
|
2149
|
+
describe("desktop skill preactivation", () => {
|
|
2150
|
+
async function preactivatedFor(turn: Record<string, unknown>): Promise<{
|
|
2151
|
+
skillIds: string[];
|
|
2152
|
+
proxyInterfaces: unknown[];
|
|
2153
|
+
skillIdsDuringLoop: string[];
|
|
2154
|
+
}> {
|
|
2155
|
+
const skillIds: string[] = [];
|
|
2156
|
+
const proxyInterfaces: unknown[] = [];
|
|
2157
|
+
let skillIdsDuringLoop: string[] = [];
|
|
2158
|
+
const fake = makeFakeConversation({
|
|
2159
|
+
processing: false,
|
|
2160
|
+
runAgentLoop: async () => {
|
|
2161
|
+
skillIdsDuringLoop = [...skillIds];
|
|
2162
|
+
},
|
|
2163
|
+
});
|
|
2164
|
+
Object.assign(fake.conversation, {
|
|
2165
|
+
addPreactivatedSkillId: (id: string) => {
|
|
2166
|
+
skillIds.push(id);
|
|
2167
|
+
},
|
|
2168
|
+
ensureHostProxiesForTurn: (sourceInterface: unknown) => {
|
|
2169
|
+
proxyInterfaces.push(sourceInterface);
|
|
2170
|
+
},
|
|
2171
|
+
});
|
|
2172
|
+
fakeConversation = fake.conversation;
|
|
2173
|
+
|
|
2174
|
+
await startVoiceTurn({
|
|
2175
|
+
...makeTurnOptions(undefined, "conv-desktop-skills"),
|
|
2176
|
+
userMessageInterface: "macos",
|
|
2177
|
+
...turn,
|
|
2178
|
+
});
|
|
2179
|
+
await flushMicrotasks();
|
|
2180
|
+
return { skillIds, proxyInterfaces, skillIdsDuringLoop };
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
|
|
2184
|
+
const result = await preactivatedFor({
|
|
2185
|
+
routingLeg: "escalated",
|
|
2186
|
+
macosDesktopSession: true,
|
|
2187
|
+
});
|
|
2188
|
+
|
|
2189
|
+
expect(result.skillIdsDuringLoop).toContain("computer-use");
|
|
2190
|
+
expect(result.proxyInterfaces).toEqual(["macos"]);
|
|
2191
|
+
});
|
|
2192
|
+
|
|
2193
|
+
test("a session from any other client leaves the desktop skills to be loaded", async () => {
|
|
2194
|
+
const result = await preactivatedFor({ routingLeg: "escalated" });
|
|
2195
|
+
|
|
2196
|
+
expect(result.skillIds).toEqual([]);
|
|
2197
|
+
expect(result.proxyInterfaces).toEqual([]);
|
|
2198
|
+
});
|
|
2199
|
+
|
|
2200
|
+
test("the toolless front-door leg is never preactivated", async () => {
|
|
2201
|
+
const result = await preactivatedFor({
|
|
2202
|
+
routingLeg: "front-door",
|
|
2203
|
+
macosDesktopSession: true,
|
|
2204
|
+
});
|
|
2205
|
+
|
|
2206
|
+
expect(result.skillIds).toEqual([]);
|
|
2207
|
+
expect(result.proxyInterfaces).toEqual([]);
|
|
2208
|
+
});
|
|
2209
|
+
});
|
|
2210
|
+
|
|
2149
2211
|
describe("cutFrontDoorContentAtVerdict", () => {
|
|
2150
2212
|
test("null when the content carries no verdict token (committed answer)", () => {
|
|
2151
2213
|
expect(
|
|
@@ -2513,6 +2575,25 @@ describe("transcript hygiene (teardown pass)", () => {
|
|
|
2513
2575
|
expect(events).toContain("loadFromDb");
|
|
2514
2576
|
});
|
|
2515
2577
|
|
|
2578
|
+
test("a row ending with a session control marker persists with it stripped", async () => {
|
|
2579
|
+
const { events } = makeReservedRowConversation();
|
|
2580
|
+
getMessageByIdImpl = () =>
|
|
2581
|
+
makeRow("Muting you for thirty seconds. [MUTE:30]");
|
|
2582
|
+
|
|
2583
|
+
await startVoiceTurn({ ...makeTurnOptions(), routingLeg: "front-door" });
|
|
2584
|
+
await flushMicrotasks();
|
|
2585
|
+
|
|
2586
|
+
expect(crudLog.updates).toEqual([
|
|
2587
|
+
{
|
|
2588
|
+
messageId: "assistant-row-1",
|
|
2589
|
+
content: JSON.stringify([
|
|
2590
|
+
{ type: "text", text: "Muting you for thirty seconds." },
|
|
2591
|
+
]),
|
|
2592
|
+
},
|
|
2593
|
+
]);
|
|
2594
|
+
expect(events).toContain("loadFromDb");
|
|
2595
|
+
});
|
|
2596
|
+
|
|
2516
2597
|
test("a mid-text [-1] (content, not command) leaves the row untouched", async () => {
|
|
2517
2598
|
const { events } = makeReservedRowConversation();
|
|
2518
2599
|
getMessageByIdImpl = () =>
|
|
@@ -16,7 +16,7 @@ import type {
|
|
|
16
16
|
ProviderResponse,
|
|
17
17
|
ToolDefinition,
|
|
18
18
|
} from "../providers/types.js";
|
|
19
|
-
import { createAbortReason } from "../util/abort-reasons.js";
|
|
19
|
+
import { createAbortReason, isAbortReason } from "../util/abort-reasons.js";
|
|
20
20
|
import { getLogger } from "../util/logger.js";
|
|
21
21
|
|
|
22
22
|
const log = getLogger("voice-progress-narration");
|
|
@@ -164,7 +164,10 @@ async function requestBoundedResponse(args: {
|
|
|
164
164
|
const timeoutTimer = setTimeout(
|
|
165
165
|
() =>
|
|
166
166
|
timeoutController.abort(
|
|
167
|
-
createAbortReason(
|
|
167
|
+
createAbortReason(
|
|
168
|
+
"voice_progress_narration_timeout",
|
|
169
|
+
"voice-progress-narration",
|
|
170
|
+
),
|
|
168
171
|
),
|
|
169
172
|
args.timeoutMs,
|
|
170
173
|
);
|
|
@@ -260,9 +263,19 @@ export function createVoiceProgressNarrator(options: {
|
|
|
260
263
|
}
|
|
261
264
|
return trimmed;
|
|
262
265
|
} catch (error) {
|
|
266
|
+
// Why the beat went unspoken, as one greppable field. A narrator whose
|
|
267
|
+
// budget is below the model's real roundtrip drops every update at the
|
|
268
|
+
// deadline and reads in the logs as a run of unrelated failures; naming
|
|
269
|
+
// the outcome is what makes that run legible as "the feature is off".
|
|
270
|
+
const outcome = isAbortReason(error)
|
|
271
|
+
? error.kind === "voice_progress_narration_timeout"
|
|
272
|
+
? "timed_out"
|
|
273
|
+
: "session_ended"
|
|
274
|
+
: "error";
|
|
263
275
|
log.info(
|
|
264
276
|
{
|
|
265
277
|
error,
|
|
278
|
+
outcome,
|
|
266
279
|
providerResolveMs,
|
|
267
280
|
totalMs: Math.round(performance.now() - startedAt),
|
|
268
281
|
timeoutMs: options.config.generationTimeoutMs,
|
|
@@ -37,6 +37,41 @@ export const ESCALATE_VERDICT_TOKEN = "[1]";
|
|
|
37
37
|
*/
|
|
38
38
|
export const MINIMIZE_ROOM_MARKER = "[-1]";
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Mic-mute session control for live voice: `[MUTE]` mutes until the user
|
|
42
|
+
* unmutes, `[MUTE:<seconds>]` mutes for that long. Like {@link END_CALL_MARKER}
|
|
43
|
+
* on a live-voice session, it is acted on only at the very end of a reply
|
|
44
|
+
* (see {@link parseTerminalSessionControl}), after the acknowledgement that
|
|
45
|
+
* precedes it has been spoken.
|
|
46
|
+
*/
|
|
47
|
+
export const MUTE_MARKER = "[MUTE]";
|
|
48
|
+
const MUTE_MARKER_PREFIX = "[MUTE:";
|
|
49
|
+
|
|
50
|
+
/** Longest timed mute a marker can ask for; longer asks mute until unmuted. */
|
|
51
|
+
export const MAX_TIMED_MUTE_SECONDS = 3600;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Progress-update cadence for the rest of a live-voice session:
|
|
55
|
+
* {@link FEWER_UPDATES_MARKER} after "don't give me updates so often",
|
|
56
|
+
* {@link NORMAL_UPDATES_MARKER} to go back. Terminal like the other session
|
|
57
|
+
* controls, but carried out by the session itself: narration is the daemon's.
|
|
58
|
+
*/
|
|
59
|
+
export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
|
|
60
|
+
export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
|
|
61
|
+
const UPDATES_MARKER_PREFIX = "[UPDATES:";
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Look session controls for live voice: turn on a screen share
|
|
65
|
+
* ({@link LOOK_SCREEN_MARKER}) or the camera ({@link LOOK_CAMERA_MARKER}) so
|
|
66
|
+
* the call can see what the user means by "take a look". Carried out by the
|
|
67
|
+
* client, like mute and end.
|
|
68
|
+
*/
|
|
69
|
+
export const LOOK_SCREEN_MARKER = "[LOOK:SCREEN]";
|
|
70
|
+
export const LOOK_CAMERA_MARKER = "[LOOK:CAMERA]";
|
|
71
|
+
/** Stop showing the call the screen and the camera, whichever is on. */
|
|
72
|
+
export const LOOK_STOP_MARKER = "[LOOK:STOP]";
|
|
73
|
+
const LOOK_MARKER_PREFIX = "[LOOK:";
|
|
74
|
+
|
|
40
75
|
// ---------------------------------------------------------------------------
|
|
41
76
|
// Regexes
|
|
42
77
|
// ---------------------------------------------------------------------------
|
|
@@ -57,6 +92,9 @@ const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
|
|
|
57
92
|
const HOLD_VERDICT_TOKEN_REGEX = /\[0\]/g;
|
|
58
93
|
const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
|
|
59
94
|
const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
|
|
95
|
+
const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
|
|
96
|
+
const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
|
|
97
|
+
const LOOK_MARKER_REGEX = /\[LOOK:\s*[^\]]*\]/g;
|
|
60
98
|
const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
|
|
61
99
|
const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
|
|
62
100
|
|
|
@@ -182,6 +220,9 @@ export function stripInternalSpeechMarkers(text: string): string {
|
|
|
182
220
|
.replace(HOLD_VERDICT_TOKEN_REGEX, "")
|
|
183
221
|
.replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
|
|
184
222
|
.replace(MINIMIZE_ROOM_MARKER_REGEX, "")
|
|
223
|
+
.replace(MUTE_MARKER_REGEX, "")
|
|
224
|
+
.replace(UPDATES_MARKER_REGEX, "")
|
|
225
|
+
.replace(LOOK_MARKER_REGEX, "")
|
|
185
226
|
.replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
|
|
186
227
|
.replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
|
|
187
228
|
return result;
|
|
@@ -207,6 +248,10 @@ const CONTROL_MARKER_STRINGS = [
|
|
|
207
248
|
"[0]",
|
|
208
249
|
"[1]",
|
|
209
250
|
"[-1]",
|
|
251
|
+
MUTE_MARKER,
|
|
252
|
+
MUTE_MARKER_PREFIX,
|
|
253
|
+
UPDATES_MARKER_PREFIX,
|
|
254
|
+
LOOK_MARKER_PREFIX,
|
|
210
255
|
"[GUARDIAN_TIMEOUT]",
|
|
211
256
|
"[GUARDIAN_UNAVAILABLE]",
|
|
212
257
|
];
|
|
@@ -220,6 +265,9 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
|
|
|
220
265
|
"[ASK_GUARDIAN:",
|
|
221
266
|
"[USER_ANSWERED:",
|
|
222
267
|
"[USER_INSTRUCTION:",
|
|
268
|
+
MUTE_MARKER_PREFIX,
|
|
269
|
+
UPDATES_MARKER_PREFIX,
|
|
270
|
+
LOOK_MARKER_PREFIX,
|
|
223
271
|
];
|
|
224
272
|
|
|
225
273
|
const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
|
|
@@ -299,3 +347,87 @@ export function createControlMarkerHoldback(
|
|
|
299
347
|
}
|
|
300
348
|
};
|
|
301
349
|
}
|
|
350
|
+
|
|
351
|
+
// ---------------------------------------------------------------------------
|
|
352
|
+
// Live-voice session controls
|
|
353
|
+
// ---------------------------------------------------------------------------
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* A session control a live-voice reply asked for with a terminal marker:
|
|
357
|
+
* `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
|
|
358
|
+
* timed form, `updates` from the progress-cadence markers, `look_screen` and
|
|
359
|
+
* `look_camera` and `look_stop` from the look markers.
|
|
360
|
+
*/
|
|
361
|
+
export type SessionControlRequest =
|
|
362
|
+
| { readonly action: "end" }
|
|
363
|
+
| { readonly action: "mute"; readonly durationMs?: number }
|
|
364
|
+
| { readonly action: "updates"; readonly cadence: "fewer" | "normal" }
|
|
365
|
+
| { readonly action: "look_screen" }
|
|
366
|
+
| { readonly action: "look_camera" }
|
|
367
|
+
| { readonly action: "look_stop" };
|
|
368
|
+
|
|
369
|
+
const TERMINAL_SESSION_CONTROL_REGEX =
|
|
370
|
+
/(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[LOOK:(SCREEN|CAMERA|STOP)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* The session control a reply ends with, or null.
|
|
374
|
+
*
|
|
375
|
+
* **Terminal position only.** The marker follows the spoken acknowledgement
|
|
376
|
+
* ("Okay, talk soon."), so a reply that merely mentions a marker mid-text, or
|
|
377
|
+
* parrots one from history before carrying on, controls nothing. This is the
|
|
378
|
+
* same rule the minimize marker's transcript hygiene applies.
|
|
379
|
+
*
|
|
380
|
+
* A timed mute whose body is not a positive number of seconds degrades to an
|
|
381
|
+
* untimed mute rather than to nothing: the user asked to be muted, and muting
|
|
382
|
+
* until they unmute is the conservative reading of a garbled duration. Asks
|
|
383
|
+
* past {@link MAX_TIMED_MUTE_SECONDS} degrade the same way.
|
|
384
|
+
*/
|
|
385
|
+
export function parseTerminalSessionControl(
|
|
386
|
+
text: string,
|
|
387
|
+
): SessionControlRequest | null {
|
|
388
|
+
const match = TERMINAL_SESSION_CONTROL_REGEX.exec(text);
|
|
389
|
+
if (!match) {
|
|
390
|
+
return null;
|
|
391
|
+
}
|
|
392
|
+
if (match[1] === END_CALL_MARKER) {
|
|
393
|
+
return { action: "end" };
|
|
394
|
+
}
|
|
395
|
+
if (match[2] !== undefined) {
|
|
396
|
+
return {
|
|
397
|
+
action: "updates",
|
|
398
|
+
cadence: match[2] === "FEWER" ? "fewer" : "normal",
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
if (match[3] !== undefined) {
|
|
402
|
+
return {
|
|
403
|
+
action:
|
|
404
|
+
match[3] === "SCREEN"
|
|
405
|
+
? "look_screen"
|
|
406
|
+
: match[3] === "CAMERA"
|
|
407
|
+
? "look_camera"
|
|
408
|
+
: "look_stop",
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
const seconds = match[4] === undefined ? NaN : Number(match[4].trim());
|
|
412
|
+
if (
|
|
413
|
+
Number.isFinite(seconds) &&
|
|
414
|
+
seconds > 0 &&
|
|
415
|
+
seconds <= MAX_TIMED_MUTE_SECONDS
|
|
416
|
+
) {
|
|
417
|
+
return { action: "mute", durationMs: Math.round(seconds * 1000) };
|
|
418
|
+
}
|
|
419
|
+
return { action: "mute" };
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Length of the marker a row's text ends with, ignoring trailing whitespace,
|
|
424
|
+
* when it is one the transcript hygiene pass strips from the persisted row
|
|
425
|
+
* (the minimize marker or a session control); 0 otherwise.
|
|
426
|
+
*/
|
|
427
|
+
export function terminalControlMarkerLength(text: string): number {
|
|
428
|
+
const trimmed = text.trimEnd();
|
|
429
|
+
if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
|
|
430
|
+
return MINIMIZE_ROOM_MARKER.length;
|
|
431
|
+
}
|
|
432
|
+
return TERMINAL_SESSION_CONTROL_REGEX.exec(trimmed)?.[1]?.length ?? 0;
|
|
433
|
+
}
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
|
|
32
32
|
import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
|
|
33
33
|
import { getOrCreateConversation } from "../daemon/conversation-store.js";
|
|
34
|
+
import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
|
|
34
35
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
35
36
|
import {
|
|
36
37
|
newestPersistedSightFrame,
|
|
@@ -70,8 +71,8 @@ import {
|
|
|
70
71
|
CALL_VERIFICATION_COMPLETE_MARKER,
|
|
71
72
|
ESCALATE_VERDICT_TOKEN,
|
|
72
73
|
HOLD_VERDICT_TOKEN,
|
|
73
|
-
MINIMIZE_ROOM_MARKER,
|
|
74
74
|
stripInternalSpeechMarkers,
|
|
75
|
+
terminalControlMarkerLength,
|
|
75
76
|
} from "./voice-control-protocol.js";
|
|
76
77
|
import {
|
|
77
78
|
createFrontDoorStreamGate,
|
|
@@ -464,6 +465,14 @@ export interface VoiceTurnOptions {
|
|
|
464
465
|
* path has always used.
|
|
465
466
|
*/
|
|
466
467
|
actorFallbackSuppressed?: boolean;
|
|
468
|
+
/**
|
|
469
|
+
* Whether the session was opened from the macOS desktop client. A leg that
|
|
470
|
+
* can run tools then starts with the desktop skills (computer use, app
|
|
471
|
+
* control, screen annotation) already active, as a typed turn from that
|
|
472
|
+
* client does. The turn interface cannot answer this: every live-voice
|
|
473
|
+
* session reports `macos` for its channel capabilities, iOS included.
|
|
474
|
+
*/
|
|
475
|
+
macosDesktopSession?: boolean;
|
|
467
476
|
/** Whether this is an inbound call (no outbound task). */
|
|
468
477
|
isInbound: boolean;
|
|
469
478
|
/** The outbound call task, if any. */
|
|
@@ -767,17 +776,22 @@ function stripMarkersFromBlocks(blocks: ContentBlock[]): ContentBlock[] {
|
|
|
767
776
|
}
|
|
768
777
|
|
|
769
778
|
/**
|
|
770
|
-
* Remove
|
|
771
|
-
*
|
|
772
|
-
*
|
|
773
|
-
*
|
|
774
|
-
*
|
|
775
|
-
*
|
|
779
|
+
* Remove a terminal control marker (the minimize marker or a session control,
|
|
780
|
+
* `markerLength` characters long) from the end of a row's text, walking text
|
|
781
|
+
* blocks from the last one backward so a marker split across block boundaries
|
|
782
|
+
* (e.g. `"Done [-"` + `"1]"`) is removed whole — the per-block strip in
|
|
783
|
+
* {@link stripMarkersFromBlocks} only sees fragments and would leave both
|
|
784
|
+
* halves in place. Callers must have established, with
|
|
785
|
+
* {@link terminalControlMarkerLength}, that the row's joined text ends with
|
|
786
|
+
* the marker after trimming trailing whitespace.
|
|
776
787
|
*/
|
|
777
|
-
function
|
|
788
|
+
function stripTerminalControlMarker(
|
|
789
|
+
blocks: ContentBlock[],
|
|
790
|
+
markerLength: number,
|
|
791
|
+
): ContentBlock[] {
|
|
778
792
|
const result = blocks.map((block) => ({ ...block }));
|
|
779
793
|
const joined = joinedTextOfBlocks(result);
|
|
780
|
-
const cutAt = joined.trimEnd().length -
|
|
794
|
+
const cutAt = joined.trimEnd().length - markerLength;
|
|
781
795
|
let blockEnd = joined.length;
|
|
782
796
|
for (let i = result.length - 1; i >= 0 && blockEnd > cutAt; i--) {
|
|
783
797
|
const block = result[i]!;
|
|
@@ -1810,14 +1824,14 @@ export async function startVoiceTurn(
|
|
|
1810
1824
|
* never the verdict token or the text streamed past the cap (issue
|
|
1811
1825
|
* #37850). A row with no spoken bridge (canned-fallback case — that
|
|
1812
1826
|
* bridge is audio-only) is deleted.
|
|
1813
|
-
* - Any leg whose row ENDS with the `[-1]` minimize marker
|
|
1814
|
-
*
|
|
1827
|
+
* - Any leg whose row ENDS with the `[-1]` minimize marker or a session
|
|
1828
|
+
* control marker (`[END_CALL]`, `[MUTE]`, `[MUTE:<seconds>]`), all
|
|
1829
|
+
* swallowed before TTS, has its text blocks rewritten through
|
|
1815
1830
|
* `stripInternalSpeechMarkers` so the marker never renders in the chat
|
|
1816
|
-
* transcript. This covers front-door answers too:
|
|
1817
|
-
*
|
|
1818
|
-
* history
|
|
1819
|
-
*
|
|
1820
|
-
* persist byte-identical.
|
|
1831
|
+
* transcript. This covers front-door answers too: a front-door answer
|
|
1832
|
+
* may end with a session control, and it can parrot `[-1]` from visible
|
|
1833
|
+
* conversation history. Deliberately scoped to terminal markers: rows
|
|
1834
|
+
* without one persist byte-identical.
|
|
1821
1835
|
*
|
|
1822
1836
|
* After a rewrite, in-memory history is reloaded from the clean DB before
|
|
1823
1837
|
* the escalated leg — blocked on this turn's teardown — snapshots it, so
|
|
@@ -1851,6 +1865,9 @@ export async function startVoiceTurn(
|
|
|
1851
1865
|
action = "delete_discarded";
|
|
1852
1866
|
} else {
|
|
1853
1867
|
const row = getMessageById(reservedAssistantRowId, opts.conversationId);
|
|
1868
|
+
const terminalMarkerLength = row
|
|
1869
|
+
? terminalControlMarkerLength(joinedTextOfBlocks(row.content))
|
|
1870
|
+
: 0;
|
|
1854
1871
|
const cut =
|
|
1855
1872
|
row && opts.routingLeg === "front-door"
|
|
1856
1873
|
? cutFrontDoorContentAtVerdict(row.content)
|
|
@@ -1869,19 +1886,18 @@ export async function startVoiceTurn(
|
|
|
1869
1886
|
action = "delete_empty";
|
|
1870
1887
|
}
|
|
1871
1888
|
} else if (
|
|
1872
|
-
// Terminal position only — mirrors
|
|
1873
|
-
//
|
|
1874
|
-
//
|
|
1875
|
-
//
|
|
1876
|
-
|
|
1877
|
-
joinedTextOfBlocks(row.content)
|
|
1878
|
-
.trimEnd()
|
|
1879
|
-
.endsWith(MINIMIZE_ROOM_MARKER)
|
|
1889
|
+
// Terminal position only — mirrors parseTerminalSessionControl: a
|
|
1890
|
+
// reply whose CONTENT contains a marker mid-text never acted on it,
|
|
1891
|
+
// so its transcript keeps that content untouched too. Front-door
|
|
1892
|
+
// answer rows (no verdict token to cut) take this branch as well.
|
|
1893
|
+
terminalMarkerLength > 0
|
|
1880
1894
|
) {
|
|
1881
1895
|
// Terminal marker first (boundary-aware — it may span text blocks),
|
|
1882
1896
|
// then the per-block strip for any interior complete markers.
|
|
1883
1897
|
const cleaned = trimOuterTextEdges(
|
|
1884
|
-
stripMarkersFromBlocks(
|
|
1898
|
+
stripMarkersFromBlocks(
|
|
1899
|
+
stripTerminalControlMarker(row.content, terminalMarkerLength),
|
|
1900
|
+
),
|
|
1885
1901
|
);
|
|
1886
1902
|
// A marker-only reply (the model said nothing beyond "[-1]") strips
|
|
1887
1903
|
// to nothing at all; keeping the row would render a blank assistant
|
|
@@ -1895,7 +1911,7 @@ export async function startVoiceTurn(
|
|
|
1895
1911
|
reservedAssistantRowId,
|
|
1896
1912
|
JSON.stringify(cleaned),
|
|
1897
1913
|
);
|
|
1898
|
-
action = "
|
|
1914
|
+
action = "strip_control_marker";
|
|
1899
1915
|
}
|
|
1900
1916
|
}
|
|
1901
1917
|
}
|
|
@@ -2009,6 +2025,20 @@ export async function startVoiceTurn(
|
|
|
2009
2025
|
(needsImagePin
|
|
2010
2026
|
? VOICE_IMAGE_PROFILE
|
|
2011
2027
|
: (conversationProfile?.profile ?? null));
|
|
2028
|
+
if (opts.macosDesktopSession === true && !frontDoorToolsSuppressed) {
|
|
2029
|
+
const sourceInterface = turnInterfaceContext.userMessageInterface;
|
|
2030
|
+
const sourceActorPrincipalId =
|
|
2031
|
+
voiceTurnValues.actorPrincipalId ?? undefined;
|
|
2032
|
+
conversation.ensureHostProxiesForTurn(
|
|
2033
|
+
sourceInterface,
|
|
2034
|
+
sourceActorPrincipalId,
|
|
2035
|
+
);
|
|
2036
|
+
preactivateHostProxySkills(
|
|
2037
|
+
conversation,
|
|
2038
|
+
sourceInterface,
|
|
2039
|
+
sourceActorPrincipalId,
|
|
2040
|
+
);
|
|
2041
|
+
}
|
|
2012
2042
|
await conversation.runAgentLoop(persistedContent, messageId, {
|
|
2013
2043
|
onEvent: (msg: AssistantEvent) => {
|
|
2014
2044
|
if (msg.type === "assistant_turn_start") {
|