@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

package/src/calls/voice-session-bridge.ts

@@ -31,6 +31,7 @@ import {
 import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
 import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
 import { getOrCreateConversation } from "../daemon/conversation-store.js";
+import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
 import type { TrustContext } from "../daemon/trust-context-types.js";
 import {
   newestPersistedSightFrame,
@@ -70,8 +71,8 @@ import {
   CALL_VERIFICATION_COMPLETE_MARKER,
   ESCALATE_VERDICT_TOKEN,
   HOLD_VERDICT_TOKEN,
-  MINIMIZE_ROOM_MARKER,
   stripInternalSpeechMarkers,
+  terminalControlMarkerLength,
 } from "./voice-control-protocol.js";
 import {
   createFrontDoorStreamGate,
@@ -188,6 +189,39 @@ function frontDoorRuleWithDigest(
   });
 }
 
+/**
+ * The triage-and-escalate rule a turn's leg adds to its control prompt: the
+ * front-door leg decides and may hand off, the escalated leg continues the
+ * answer after a holding phrase was already spoken. Null when routing is off.
+ * One rule for both prompt shapes, the auto-built phone prompt and a
+ * caller-supplied one, so the front-door model is anchored to the same
+ * caller words on every transport.
+ */
+function routingLegRuleFor(
+  opts: Pick<
+    VoiceTurnOptions,
+    | "routingLeg"
+    | "unifiedVerdict"
+    | "spokenEscalationBridge"
+    | "directEscalated"
+  >,
+  callerUtterance: string,
+): string | null {
+  switch (opts.routingLeg) {
+    case "front-door":
+      return frontDoorRuleWithDigest(
+        opts.unifiedVerdict === true,
+        callerUtterance,
+      );
+    case "escalated":
+      return opts.directEscalated === true
+        ? null
+        : escalatedContinuationRule(opts.spokenEscalationBridge);
+    default:
+      return null;
+  }
+}
+
 /**
  * Exact message thrown when `opts.signal` aborts while the turn is waiting
  * for the conversation to become available. The call controller's abort
@@ -329,6 +363,7 @@ export interface VoiceRunEventSink {
     toolName: string,
     input: Record<string, unknown>,
     toolUseId?: string,
+    allowedToolNames?: ReadonlySet<string>,
   ): void;
   onToolResult(event: VoiceToolResultEvent): void;
 }
@@ -343,7 +378,11 @@ export interface VoiceTurnCallbacks {
   /** Fired when the agent run starts a definitive tool use this turn. */
   tool_use_start?: (
     toolName: string,
-    detail?: { toolUseId?: string; input?: Record<string, unknown> },
+    detail?: {
+      toolUseId?: string;
+      input?: Record<string, unknown>;
+      allowedToolNames?: ReadonlySet<string>;
+    },
   ) => void;
   /** Fired when a tool invocation finishes. */
   tool_result?: (event: VoiceToolResultEvent) => void;
@@ -373,8 +412,9 @@ export interface VoiceTurnOptions {
    *
    * Deliberately separate from {@link userMessageInterface}: that field feeds
    * `resolveChannelCapabilities` and decides what the turn may do, so it is not
-   * free to carry attribution. Absent for phone calls and for clients that send
-   * no identity on the start frame.
+   * free to carry attribution. A phone call passes its call session id and a
+   * `phone_*` entry naming the call's direction, with no client. Absent for
+   * clients that send no identity on the start frame.
    */
   voiceTelemetry?: {
     sessionId: string;
@@ -435,6 +475,14 @@ export interface VoiceTurnOptions {
    * path has always used.
    */
   actorFallbackSuppressed?: boolean;
+  /**
+   * Whether the session was opened from the macOS desktop client. A leg that
+   * can run tools then starts with the desktop skills (computer use, app
+   * control, screen annotation) already active, as a typed turn from that
+   * client does. The turn interface cannot answer this: every live-voice
+   * session reports `macos` for its channel capabilities, iOS included.
+   */
+  macosDesktopSession?: boolean;
   /** Whether this is an inbound call (no outbound task). */
   isInbound: boolean;
   /** The outbound call task, if any. */
@@ -447,7 +495,7 @@ export interface VoiceTurnOptions {
   onComplete?: () => void;
   /** Called when the agent loop encounters an error. */
   onError?: (message: string) => void;
-  /** Event-name callbacks used by non-phone voice clients. */
+  /** Event-name callbacks: tool activity, persisted row ids, raw stream. */
   callbacks?: VoiceTurnCallbacks;
   /**
    * Called when this turn leaves a confirmation for the user to answer instead
@@ -482,10 +530,9 @@ export interface VoiceTurnOptions {
    */
   overrideProfile?: string;
   /**
-   * Which leg of a triaged turn this is, so the auto-built phone control prompt
-   * can add the front-door triage rule or the escalated continuation rule.
-   * Undefined = routing off; no routing rules are added. Ignored when a caller
-   * supplies its own `voiceControlPrompt`.
+   * Which leg of a triaged turn this is, so the control prompt (auto-built or
+   * caller-supplied) carries the front-door triage rule or the escalated
+   * continuation rule. Undefined = routing off; no routing rules are added.
    */
   routingLeg?: VoiceRoutingLeg;
   /**
@@ -496,6 +543,8 @@ export interface VoiceTurnOptions {
    * Only meaningful with `routingLeg: "escalated"`.
    */
   spokenEscalationBridge?: string;
+  /** Run the strong leg directly, without claiming a holding phrase was spoken. */
+  directEscalated?: boolean;
   /**
    * Marks this turn's `content` as an internal instruction rather than user
    * speech: it persists `hidden` so `/messages` filters it after a reload,
@@ -611,9 +660,8 @@ function buildVoiceCallControlPrompt(opts: {
   task?: string | null;
   isCallerGuardian?: boolean;
   skipDisclosure?: boolean;
-  routingLeg?: VoiceRoutingLeg;
-  spokenEscalationBridge?: string;
-  unifiedVerdict?: boolean;
+  /** The turn's routing-leg rule (see {@link routingLegRuleFor}), if any. */
+  routingLegRule?: string | null;
 }): string {
   const config = getConfig();
   const disclosureEnabled =
@@ -665,22 +713,18 @@ function buildVoiceCallControlPrompt(opts: {
     );
   }
 
-  lines.push(
-    '6. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
-  );
-
   if (opts.isInbound) {
     if (opts.isCallerGuardian) {
       lines.push(
-        '7. If the latest user turn is "(call connected — deliver opening greeting)", this is your user calling you. Answer casually and briefly, like picking up a call from someone you know well. For example: "Hey!" or "What\'s up?" Do NOT introduce yourself, do NOT say you are calling on behalf of anyone, and do NOT ask how you can help in a formal way. Keep it short and natural.',
+        '6. If the latest user turn is "(call connected — deliver opening greeting)", this is your user calling you. Answer casually and briefly, like picking up a call from someone you know well. For example: "Hey!" or "What\'s up?" Do NOT introduce yourself, do NOT say you are calling on behalf of anyone, and do NOT ask how you can help in a formal way. Keep it short and natural.',
       );
     } else {
       lines.push(
-        '7. If the latest user turn is "(call connected — deliver opening greeting)", this is an inbound call you are answering (not a call you initiated). Greet the caller warmly and ask how you can help. Introduce yourself once at the start using your assistant name if you know it (for example: "Hey there, this is Ava, Sam\'s assistant. How can I help?"). If your assistant name is not known, skip the name and just identify yourself as the guardian\'s assistant. Never use a UUID-shaped internal assistant ID as your spoken name. Do NOT say "I\'m calling" or "I\'m calling on behalf of". Vary the wording; do not use a fixed template.',
+        '6. If the latest user turn is "(call connected — deliver opening greeting)", this is an inbound call you are answering (not a call you initiated). Greet the caller warmly and ask how you can help. Introduce yourself once at the start using your assistant name if you know it (for example: "Hey there, this is Ava, Sam\'s assistant. How can I help?"). If your assistant name is not known, skip the name and just identify yourself as the guardian\'s assistant. Never use a UUID-shaped internal assistant ID as your spoken name. Do NOT say "I\'m calling" or "I\'m calling on behalf of". Vary the wording; do not use a fixed template.',
       );
     }
     lines.push(
-      "8. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.",
+      "7. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.",
     );
   } else {
     const disclosureReminder =
@@ -688,27 +732,22 @@ function buildVoiceCallControlPrompt(opts: {
         ? " However, the disclosure text from rule 0 is separate from self-introduction and must always be included in your opening greeting, even if the Task does not mention introducing yourself."
         : "";
     lines.push(
-      '7. If the latest user turn is "(verification completed — transitioning into conversation)", the caller just completed a phone verification code challenge on this call. Greet them naturally and ask if there is anything you can help with. Keep it casual and brief.',
-      `If the latest user turn is "(call connected — deliver opening greeting)", deliver your opening greeting based solely on the Task context above. The Task already describes how to open the call — follow it directly without adding any extra introduction on top. If the Task says to introduce yourself, do so once. If the Task does not mention introducing yourself, skip the introduction.${disclosureReminder} Vary the wording naturally; do not use a fixed template.`,
-      "8. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.",
+      '6. If the latest user turn is "(verification completed — transitioning into conversation)", the caller just completed a phone verification code challenge on this call. Greet them naturally and ask if there is anything you can help with. Keep it casual and brief.',
+      `If the latest user turn is "(call connected — deliver opening greeting)", deliver your opening greeting based solely on the Task context above. The Task already describes how to open the call, so follow it directly without adding any extra introduction on top. If the Task says to introduce yourself, do so once. If the Task does not mention introducing yourself, skip the introduction.${disclosureReminder} Vary the wording naturally; do not use a fixed template.`,
+      "7. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.",
     );
   }
 
   lines.push(
-    "9. After the opening greeting turn, treat the Task field as background context only — do not re-execute its instructions on subsequent turns.",
-    '10. Do not make up information. If you are unsure, use [ASK_GUARDIAN: your question] to consult your guardian. For tool permission requests, use [ASK_GUARDIAN_APPROVAL: {"question":"...","toolName":"...","input":{...}}].',
-    `11. ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Protocol markers like ${opts.isCallerGuardian ? "[END_CALL]" : "[ASK_GUARDIAN: ...] and [END_CALL]"} are not spoken text and should still be used normally.`,
-    `12. Speak the caller's language: reply in the language of the caller's most recent actual speech, and follow them if they switch languages mid-call. Synthetic user turns (parenthetical markers like the call-connected and verification-completed notices) are not caller speech and never set the language. Before the caller has spoken, such as on the opening greeting turn, ${preSpeechLanguageRuleFragment(config.services.stt.language, sttCatalogKeyForRole(config.services.stt, "telephony"))}.`,
-    `13. ${PHONE_NO_SETUP_FLOWS_RULE}`,
+    "8. After the opening greeting turn, treat the Task field as background context only. Do not re-execute its instructions on subsequent turns.",
+    '9. Do not make up information. If you are unsure, use [ASK_GUARDIAN: your question] to consult your guardian. For tool permission requests, use [ASK_GUARDIAN_APPROVAL: {"question":"...","toolName":"...","input":{...}}].',
+    `10. ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Protocol markers like ${opts.isCallerGuardian ? "[END_CALL]" : "[ASK_GUARDIAN: ...] and [END_CALL]"} are not spoken text and should still be used normally.`,
+    `11. Speak the caller's language: reply in the language of the caller's most recent actual speech, and follow them if they switch languages mid-call. Synthetic user turns (parenthetical markers like the call-connected and verification-completed notices) are not caller speech and never set the language. Before the caller has spoken, such as on the opening greeting turn, ${preSpeechLanguageRuleFragment(config.services.stt.language, sttCatalogKeyForRole(config.services.stt, "telephony"))}.`,
+    `12. ${PHONE_NO_SETUP_FLOWS_RULE}`,
   );
 
-  // Triage-and-escalate routing rules. The front-door leg decides and may
-  // hand off; the escalated leg continues the answer after a holding phrase
-  // was already spoken.
-  if (opts.routingLeg === "front-door") {
-    lines.push(`14. ${frontDoorRuleWithDigest(opts.unifiedVerdict === true)}`);
-  } else if (opts.routingLeg === "escalated") {
-    lines.push(`14. ${escalatedContinuationRule(opts.spokenEscalationBridge)}`);
+  if (opts.routingLegRule) {
+    lines.push(`13. ${opts.routingLegRule}`);
   }
 
   lines.push("</voice_call_control>");
@@ -749,17 +788,22 @@ function stripMarkersFromBlocks(blocks: ContentBlock[]): ContentBlock[] {
 }
 
 /**
- * Remove the terminal MINIMIZE_ROOM_MARKER from the end of a row's text,
- * walking text blocks from the last one backward so a marker split across
- * block boundaries (e.g. `"Done [-"` + `"1]"`) is removed whole — the
- * per-block strip in {@link stripMarkersFromBlocks} only sees fragments and
- * would leave both halves in place. Callers must have established that the
- * row's joined text ends with the marker after trimming trailing whitespace.
+ * Remove a terminal control marker (the minimize marker or a session control,
+ * `markerLength` characters long) from the end of a row's text, walking text
+ * blocks from the last one backward so a marker split across block boundaries
+ * (e.g. `"Done [-"` + `"1]"`) is removed whole — the per-block strip in
+ * {@link stripMarkersFromBlocks} only sees fragments and would leave both
+ * halves in place. Callers must have established, with
+ * {@link terminalControlMarkerLength}, that the row's joined text ends with
+ * the marker after trimming trailing whitespace.
  */
-function stripTerminalMinimizeMarker(blocks: ContentBlock[]): ContentBlock[] {
+function stripTerminalControlMarker(
+  blocks: ContentBlock[],
+  markerLength: number,
+): ContentBlock[] {
   const result = blocks.map((block) => ({ ...block }));
   const joined = joinedTextOfBlocks(result);
-  const cutAt = joined.trimEnd().length - MINIMIZE_ROOM_MARKER.length;
+  const cutAt = joined.trimEnd().length - markerLength;
   let blockEnd = joined.length;
   for (let i = result.length - 1; i >= 0 && blockEnd > cutAt; i--) {
     const block = result[i]!;
@@ -889,9 +933,13 @@ export async function startVoiceTurn(
     onError: (message) => {
       opts.onError?.(message);
     },
-    onToolUse: (toolName, input, toolUseId) => {
+    onToolUse: (toolName, input, toolUseId, allowedToolNames) => {
       log.debug({ toolName, input }, "Voice turn tool_use event");
-      opts.callbacks?.tool_use_start?.(toolName, { toolUseId, input });
+      opts.callbacks?.tool_use_start?.(toolName, {
+        toolUseId,
+        input,
+        ...(allowedToolNames !== undefined ? { allowedToolNames } : {}),
+      });
     },
     onToolResult: (event) => {
       opts.callbacks?.tool_result?.(event);
@@ -956,6 +1004,10 @@ export async function startVoiceTurn(
   // control markers (ASK_GUARDIAN, END_CALL, etc.) and recognize opener turns.
   const isCallerGuardian = opts.trustContext?.trustClass === "guardian";
 
+  // The front-door rule anchors on the words the model sees as the user
+  // turn, which for a phone sentinel is its neutral persisted form rather
+  // than the marker itself.
+  const routingLegRule = routingLegRuleFor(opts, persistedContent);
   let voiceCallControlPrompt: string | null;
   if (opts.voiceControlPrompt === undefined) {
     voiceCallControlPrompt = buildVoiceCallControlPrompt({
@@ -963,23 +1015,14 @@ export async function startVoiceTurn(
       task: opts.task,
       isCallerGuardian,
       skipDisclosure: opts.skipDisclosure,
-      routingLeg: opts.routingLeg,
-      spokenEscalationBridge: opts.spokenEscalationBridge,
-      unifiedVerdict: opts.unifiedVerdict,
+      routingLegRule,
     });
   } else {
-    // A caller-supplied prompt (e.g. live-voice) bypasses
-    // buildVoiceCallControlPrompt, which is where the triage-and-escalate rule
-    // is normally injected from `routingLeg`. Append it here too — without it
-    // the front-door leg would run on the fast profile but never learn the
-    // verdict protocol, so it could not hold or hand off to the escalated leg.
+    // A caller-supplied prompt (live voice) replaces the phone prompt, so the
+    // routing-leg rule is appended to it here. Without it the front-door leg
+    // would run on the fast profile but never learn the verdict protocol, so
+    // it could not hold or hand off to the escalated leg.
     voiceCallControlPrompt = opts.voiceControlPrompt;
-    const routingLegRule =
-      opts.routingLeg === "front-door"
-        ? frontDoorRuleWithDigest(opts.unifiedVerdict === true, opts.content)
-        : opts.routingLeg === "escalated"
-          ? escalatedContinuationRule(opts.spokenEscalationBridge)
-          : null;
     if (voiceCallControlPrompt != null && routingLegRule) {
       voiceCallControlPrompt = `${voiceCallControlPrompt}\n\n${routingLegRule}`;
     }
@@ -1797,14 +1840,14 @@ export async function startVoiceTurn(
    *   never the verdict token or the text streamed past the cap (issue
    *   #37850). A row with no spoken bridge (canned-fallback case — that
    *   bridge is audio-only) is deleted.
-   * - Any leg whose row ENDS with the `[-1]` minimize marker (swallowed
-   *   before TTS on the live path) has its text blocks rewritten through
+   * - Any leg whose row ENDS with the `[-1]` minimize marker or a session
+   *   control marker (`[END_CALL]`, `[MUTE]`, `[MUTE:<seconds>]`), all
+   *   swallowed before TTS, has its text blocks rewritten through
    *   `stripInternalSpeechMarkers` so the marker never renders in the chat
-   *   transcript. This covers front-door answers too: that leg is never
-   *   taught the marker, but it can parrot one from visible conversation
-   *   history, and the parroted marker is never spoken and never minimizes
-   *   the room. Deliberately scoped to that marker: rows without it
-   *   persist byte-identical.
+   *   transcript. This covers front-door answers too: a front-door answer
+   *   may end with a session control, and it can parrot `[-1]` from visible
+   *   conversation history. Deliberately scoped to terminal markers: rows
+   *   without one persist byte-identical.
    *
    * After a rewrite, in-memory history is reloaded from the clean DB before
    * the escalated leg — blocked on this turn's teardown — snapshots it, so
@@ -1838,6 +1881,9 @@ export async function startVoiceTurn(
         action = "delete_discarded";
       } else {
         const row = getMessageById(reservedAssistantRowId, opts.conversationId);
+        const terminalMarkerLength = row
+          ? terminalControlMarkerLength(joinedTextOfBlocks(row.content))
+          : 0;
         const cut =
           row && opts.routingLeg === "front-door"
             ? cutFrontDoorContentAtVerdict(row.content)
@@ -1856,19 +1902,18 @@ export async function startVoiceTurn(
             action = "delete_empty";
           }
         } else if (
-          // Terminal position only — mirrors the live latch in
-          // createControlMarkerHoldback: a reply whose CONTENT contains
-          // "[-1]" mid-text never minimized the room, so its transcript
-          // keeps that content untouched too. Front-door answer rows (no
-          // verdict token to cut) take this branch as well.
-          joinedTextOfBlocks(row.content)
-            .trimEnd()
-            .endsWith(MINIMIZE_ROOM_MARKER)
+          // Terminal position only — mirrors parseTerminalSessionControl: a
+          // reply whose CONTENT contains a marker mid-text never acted on it,
+          // so its transcript keeps that content untouched too. Front-door
+          // answer rows (no verdict token to cut) take this branch as well.
+          terminalMarkerLength > 0
         ) {
           // Terminal marker first (boundary-aware — it may span text blocks),
           // then the per-block strip for any interior complete markers.
           const cleaned = trimOuterTextEdges(
-            stripMarkersFromBlocks(stripTerminalMinimizeMarker(row.content)),
+            stripMarkersFromBlocks(
+              stripTerminalControlMarker(row.content, terminalMarkerLength),
+            ),
           );
           // A marker-only reply (the model said nothing beyond "[-1]") strips
           // to nothing at all; keeping the row would render a blank assistant
@@ -1882,7 +1927,7 @@ export async function startVoiceTurn(
               reservedAssistantRowId,
               JSON.stringify(cleaned),
             );
-            action = "strip_minimize_marker";
+            action = "strip_control_marker";
           }
         }
       }
@@ -1996,6 +2041,20 @@ export async function startVoiceTurn(
         (needsImagePin
           ? VOICE_IMAGE_PROFILE
           : (conversationProfile?.profile ?? null));
+      if (opts.macosDesktopSession === true && !frontDoorToolsSuppressed) {
+        const sourceInterface = turnInterfaceContext.userMessageInterface;
+        const sourceActorPrincipalId =
+          voiceTurnValues.actorPrincipalId ?? undefined;
+        conversation.ensureHostProxiesForTurn(
+          sourceInterface,
+          sourceActorPrincipalId,
+        );
+        preactivateHostProxySkills(
+          conversation,
+          sourceInterface,
+          sourceActorPrincipalId,
+        );
+      }
       await conversation.runAgentLoop(persistedContent, messageId, {
         onEvent: (msg: AssistantEvent) => {
           if (msg.type === "assistant_turn_start") {
@@ -2038,7 +2097,12 @@ export async function startVoiceTurn(
           } else if (msg.type === "conversation_error") {
             eventSink.onError(msg.userMessage);
           } else if (msg.type === "tool_use_start") {
-            eventSink.onToolUse(msg.toolName, msg.input, msg.toolUseId);
+            eventSink.onToolUse(
+              msg.toolName,
+              msg.input,
+              msg.toolUseId,
+              conversation.allowedToolNames,
+            );
           } else if (msg.type === "tool_result") {
             eventSink.onToolResult({
               toolName: msg.toolName,