@vellumai/assistant 0.11.10-dev.202609091714.3d3ebb3 → 0.11.10-dev.202609091820.7951ec9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.10-dev.202609091714.3d3ebb3",
+  "version": "0.11.10-dev.202609091820.7951ec9",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/calls/__tests__/voice-session-bridge.test.ts

@@ -12,6 +12,7 @@
  * covered by `src/__tests__/conversation-wait-for-idle.test.ts`.
  */
 import {
+  afterEach,
   beforeEach,
   describe,
   expect,
@@ -35,8 +36,18 @@ mock.module("../../daemon/conversation-store.js", () => ({
 // production (a BYO provider resolves the profile key through its own column
 // of the intent matrix), so it is scripted rather than read from a catalog.
 let pinProfileSupportsVision = true;
+// Vision capability of the conversation's own profile, which an escalated
+// leg is pinned to. Scripted for the same reason.
+let conversationProfileSupportsVision = true;
+// Per-profile answers that outrank the two switches above, for a mix whose
+// arms differ.
+const visionByProfile = new Map<string, boolean>();
 mock.module("../../plugin-api/vision-support.js", () => ({
-  doesSupportVision: () => pinProfileSupportsVision,
+  doesSupportVision: (profile: string) =>
+    visionByProfile.get(profile) ??
+    (profile === "latency-optimized"
+      ? pinProfileSupportsVision
+      : conversationProfileSupportsVision),
 }));
 
 // Attachment hydration for the parked-camera-frame path. Only `att-frame-*`
@@ -126,6 +137,8 @@ mock.module("../../persistence/conversation-crud.js", () => ({
 }));
 
 import { setConfig } from "../../__tests__/helpers/set-config.js";
+import { selectWinningProfile } from "../../config/llm-resolver.js";
+import { getConfig } from "../../config/loader.js";
 import { ABORT_WATCHDOG_MS } from "../../daemon/abort-watchdog.js";
 import { VOICE_ESCALATION_CONTINUATION_MESSAGE_KIND } from "../../plugin-api/constants.js";
 import { assistantEventHub } from "../../runtime/assistant-event-hub.js";
@@ -162,6 +175,9 @@ interface WaitForIdleCall {
 
 interface FakeConversation {
   conversationId: string;
+  /** Per-conversation profile pin an escalated leg follows; absent = none. */
+  inferenceProfile?: string | null;
+  inferenceProfileExpiresAt?: number | null;
   callSessionId: string | undefined;
   forcePromptSideEffects: boolean;
   currentRequestId: string | undefined;
@@ -208,6 +224,10 @@ function makeFakeConversation(opts: {
   workingDir?: string;
   /** In-memory history the profile pin reads; undefined models a text-only call. */
   messages?: Array<{ role: string; content: unknown[] }>;
+  /** Per-conversation profile pin; undefined models an unpinned conversation. */
+  inferenceProfile?: string;
+  /** Expiry of that pin; a past stamp models a lapsed session pin. */
+  inferenceProfileExpiresAt?: number;
 }) {
   const waitForIdleCalls: WaitForIdleCall[] = [];
   const confirmationDecisions: Array<{ requestId: string; decision: string }> =
@@ -224,6 +244,12 @@ function makeFakeConversation(opts: {
     | undefined;
   const conversation: FakeConversation = {
     conversationId: "conv-voice-bridge-test",
+    ...(opts.inferenceProfile !== undefined
+      ? { inferenceProfile: opts.inferenceProfile }
+      : {}),
+    ...(opts.inferenceProfileExpiresAt !== undefined
+      ? { inferenceProfileExpiresAt: opts.inferenceProfileExpiresAt }
+      : {}),
     // The workspace boundary the reach check compares paths against. A real
     // conversation always has one; without it the approval gate fails closed.
     workingDir: opts.workingDir ?? "/tmp/workspace-voice-bridge-test",
@@ -2581,19 +2607,19 @@ describe("transcript hygiene (teardown pass)", () => {
   });
 });
 
-describe("startVoiceTurn image-bearing profile pin", () => {
-  /** A persisted user message carrying a photo taken mid-call. */
-  const PHOTO_HISTORY = [
-    { role: "user", content: [{ type: "text", text: "here's a photo:" }] },
-    {
-      role: "user",
-      content: [
-        { type: "text", text: "here's a photo:" },
-        { type: "image", source: { type: "base64", data: "abc" } },
-      ],
-    },
-  ];
+/** A persisted user message carrying a photo taken mid-call. */
+const PHOTO_HISTORY = [
+  { role: "user", content: [{ type: "text", text: "here's a photo:" }] },
+  {
+    role: "user",
+    content: [
+      { type: "text", text: "here's a photo:" },
+      { type: "image", source: { type: "base64", data: "abc" } },
+    ],
+  },
+];
 
+describe("startVoiceTurn image-bearing profile pin", () => {
   beforeEach(() => {
     pinProfileSupportsVision = true;
   });
@@ -2682,3 +2708,188 @@ describe("startVoiceTurn image-bearing profile pin", () => {
     expect(runOptions.overrideProfile).toBe("quality-optimized");
   });
 });
+
+describe("startVoiceTurn escalated-leg profile pin", () => {
+  beforeEach(() => {
+    pinProfileSupportsVision = true;
+    conversationProfileSupportsVision = true;
+  });
+
+  afterEach(() => {
+    setConfig("llm", {});
+  });
+
+  async function runOptionsFor(opts: {
+    messages?: Array<{ role: string; content: unknown[] }>;
+    inferenceProfile?: string;
+    inferenceProfileExpiresAt?: number;
+    turn?: Record<string, unknown>;
+  }): Promise<Record<string, unknown>> {
+    const fake = makeFakeConversation({
+      processing: false,
+      ...(opts.messages ? { messages: opts.messages } : {}),
+      ...(opts.inferenceProfile
+        ? { inferenceProfile: opts.inferenceProfile }
+        : {}),
+      ...(opts.inferenceProfileExpiresAt !== undefined
+        ? { inferenceProfileExpiresAt: opts.inferenceProfileExpiresAt }
+        : {}),
+    });
+    fakeConversation = fake.conversation;
+    let runOptions: Record<string, unknown> = {};
+    fake.conversation.runAgentLoop = async (...args: unknown[]) => {
+      runOptions = args[2] as Record<string, unknown>;
+    };
+    await startVoiceTurn({
+      ...makeTurnOptions(),
+      routingLeg: "escalated",
+      ...(opts.turn ?? {}),
+    });
+    return runOptions;
+  }
+
+  test("with no chat-model selection the leg keeps the call-site profile", async () => {
+    const runOptions = await runOptionsFor({});
+
+    expect(runOptions.callSite).toBe("callAgent");
+    expect(runOptions.overrideProfile).toBeUndefined();
+    expect(runOptions.forceOverrideProfile).toBeUndefined();
+  });
+
+  test("the workspace chat-model selection pins the leg", async () => {
+    // `callAgent`'s chain never consults `llm.activeProfile`, so without the
+    // pin the hand-off would land on the site's shipped default while the
+    // same conversation's typed turns run on the user's pick.
+    setConfig("llm", { activeProfile: "quality-optimized" });
+
+    const runOptions = await runOptionsFor({});
+
+    expect(runOptions.callSite).toBe("callAgent");
+    expect(runOptions.overrideProfile).toBe("quality-optimized");
+    expect(runOptions.forceOverrideProfile).toBe(true);
+  });
+
+  test("the conversation's own pin wins over the workspace selection", async () => {
+    setConfig("llm", { activeProfile: "quality-optimized" });
+
+    const runOptions = await runOptionsFor({
+      inferenceProfile: "cost-optimized",
+    });
+
+    expect(runOptions.overrideProfile).toBe("cost-optimized");
+    expect(runOptions.forceOverrideProfile).toBe(true);
+  });
+
+  test("a lapsed conversation pin falls back to the workspace selection", async () => {
+    setConfig("llm", { activeProfile: "quality-optimized" });
+
+    const runOptions = await runOptionsFor({
+      inferenceProfile: "cost-optimized",
+      inferenceProfileExpiresAt: Date.now() - 1,
+    });
+
+    expect(runOptions.overrideProfile).toBe("quality-optimized");
+  });
+
+  test("only the escalated leg follows the conversation", async () => {
+    setConfig("llm", { activeProfile: "quality-optimized" });
+
+    const frontDoor = await runOptionsFor({
+      turn: { routingLeg: "front-door" },
+    });
+    expect(frontDoor.callSite).toBe("voiceFrontDoor");
+    expect(frontDoor.overrideProfile).toBeUndefined();
+
+    const unrouted = await runOptionsFor({ turn: { routingLeg: undefined } });
+    expect(unrouted.callSite).toBe("callAgent");
+    expect(unrouted.overrideProfile).toBeUndefined();
+  });
+
+  test("an image stays on a conversation profile whose model takes it", async () => {
+    setConfig("llm", { activeProfile: "quality-optimized" });
+
+    const runOptions = await runOptionsFor({ messages: PHOTO_HISTORY });
+
+    expect(runOptions.overrideProfile).toBe("quality-optimized");
+  });
+
+  test("an image hands a text-only conversation profile to the image pin", async () => {
+    // A model that rejects the image fails the whole leg, so the image pin
+    // outranks the conversation's choice for this one turn.
+    setConfig("llm", { activeProfile: "quality-optimized" });
+    conversationProfileSupportsVision = false;
+
+    const runOptions = await runOptionsFor({ messages: PHOTO_HISTORY });
+
+    expect(runOptions.overrideProfile).toBe("latency-optimized");
+    expect(runOptions.forceOverrideProfile).toBe(true);
+  });
+
+  test("an image with no image-capable profile anywhere keeps the conversation profile", async () => {
+    setConfig("llm", { activeProfile: "quality-optimized" });
+    conversationProfileSupportsVision = false;
+    pinProfileSupportsVision = false;
+
+    const runOptions = await runOptionsFor({ messages: PHOTO_HISTORY });
+
+    expect(runOptions.overrideProfile).toBe("quality-optimized");
+  });
+
+  test("an explicit routing pin wins over the conversation profile", async () => {
+    setConfig("llm", { activeProfile: "quality-optimized" });
+
+    const runOptions = await runOptionsFor({
+      turn: { overrideProfile: "balanced" },
+    });
+
+    expect(runOptions.overrideProfile).toBe("balanced");
+  });
+
+  test("a mix is judged by the arm serving this conversation, not by any arm", async () => {
+    // A mix reads as vision-capable when any arm is, but dispatch expands it
+    // to one arm from the conversation seed. Only that arm's model sees the
+    // image, so only that arm's capability decides whether the image pin
+    // takes over. The pin itself stays the mix's own name, so dispatch lands
+    // on the same arm.
+    const llm = {
+      activeProfile: "voice-mix",
+      profiles: {
+        "voice-mix": {
+          mix: [
+            { profile: "quality-optimized", weight: 1 },
+            { profile: "cost-optimized", weight: 1 },
+          ],
+        },
+      },
+    };
+    setConfig("llm", llm);
+    let chosenArm: string | undefined;
+    selectWinningProfile("mainAgent", getConfig().llm, {
+      selectionSeed: "conv-voice-bridge-test",
+      onMixSelected: ({ chosenProfile }) => {
+        chosenArm = chosenProfile;
+      },
+    });
+    expect(chosenArm).toBeDefined();
+    const otherArm =
+      chosenArm === "quality-optimized"
+        ? "cost-optimized"
+        : "quality-optimized";
+    try {
+      // Only the unchosen arm takes images: judged as "any arm", the mix
+      // would keep the pin off and the image would reach a text-only model.
+      visionByProfile.set(chosenArm!, false);
+      visionByProfile.set(otherArm, true);
+      const textOnlyArm = await runOptionsFor({ messages: PHOTO_HISTORY });
+      expect(textOnlyArm.overrideProfile).toBe("latency-optimized");
+
+      // Only the chosen arm takes images: no pin needed, the mix stands.
+      visionByProfile.set(chosenArm!, true);
+      visionByProfile.set(otherArm, false);
+      const visionArm = await runOptionsFor({ messages: PHOTO_HISTORY });
+      expect(visionArm.overrideProfile).toBe("voice-mix");
+    } finally {
+      visionByProfile.clear();
+    }
+  });
+});

package/src/calls/voice-session-bridge.ts

@@ -22,6 +22,7 @@ import type {
   TurnChannelContext,
   TurnInterfaceContext,
 } from "../channels/types.js";
+import { selectWinningProfile } from "../config/llm-resolver.js";
 import { getConfig } from "../config/loader.js";
 import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
 import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
@@ -37,7 +38,9 @@ import { resolveAttachmentsForPersist } from "../persistence/attachments-store.j
 import {
   deleteMessageById,
   getMessageById,
+  type OverrideProfileFields,
   recordConversationPersistedSeq,
+  resolveOverrideProfile,
   updateMessageContent,
 } from "../persistence/conversation-crud.js";
 import { VOICE_ESCALATION_CONTINUATION_MESSAGE_KIND } from "../plugin-api/constants.js";
@@ -91,6 +94,50 @@ const log = getLogger("voice-session-bridge");
  */
 const VOICE_IMAGE_PROFILE = "latency-optimized";
 
+/**
+ * The profile the conversation's own text turns run on, resolved the way a
+ * `mainAgent` turn resolves it: the conversation's pinned profile when it
+ * carries one, else the workspace chat-model selection (`llm.activeProfile`),
+ * else the main agent's call-site pin.
+ *
+ * The escalated voice leg runs through `callAgent`, whose chain never
+ * consults `llm.activeProfile`, so without this the hand-off lands on that
+ * site's shipped `balanced` default while the same conversation's typed
+ * turns run on whatever the user picked. Pinning the text-turn winner keeps
+ * the stronger model the front door escalates to the one the conversation is
+ * already using.
+ *
+ * `profile` is the name to pin (a mix's own name, so dispatch re-expands it
+ * to the same arm from the conversation seed); `modelProfile` is the concrete
+ * profile whose model actually runs (the chosen arm of a mix), which is what
+ * capability checks must judge: a mix reads as vision-capable when any arm
+ * is, but only one arm serves this conversation.
+ *
+ * Null when nothing above named a profile (the winner is the code-owned
+ * anchor): the leg then keeps its ordinary call-site resolution, which lands
+ * on the same anchor intent and still honors a `callAgent` site pin.
+ */
+function conversationProfileForEscalation(
+  conversation: OverrideProfileFields & { conversationId: string },
+): { profile: string; modelProfile: string } | null {
+  const overrideProfile = resolveOverrideProfile(conversation);
+  let chosenArm: string | undefined;
+  const selection = selectWinningProfile("mainAgent", getConfig().llm, {
+    ...(overrideProfile != null ? { overrideProfile } : {}),
+    selectionSeed: conversation.conversationId,
+    onMixSelected: ({ chosenProfile }) => {
+      chosenArm = chosenProfile;
+    },
+  });
+  if (selection.source === "default" || selection.profileName == null) {
+    return null;
+  }
+  return {
+    profile: selection.profileName,
+    modelProfile: chosenArm ?? selection.profileName,
+  };
+}
+
 /**
  * Does this conversation's history carry an image?
  *
@@ -422,10 +469,10 @@ export interface VoiceTurnOptions {
   signal?: AbortSignal;
   /**
    * Ad-hoc inference-profile override applied to every LLM call this turn
-   * issues (forwarded to `runAgentLoop` with `forceOverrideProfile`). Used by
-   * triage-and-escalate voice routing to run the front-door leg on the fast
-   * profile and the escalated leg on the quality profile. Undefined = the
-   * call-site default (today's behavior).
+   * issues (forwarded to `runAgentLoop` with `forceOverrideProfile`). Wins
+   * over the bridge's own pins (the image pin, and the conversation's
+   * profile for an escalated leg). Undefined = those pins, else the
+   * call-site default.
    */
   overrideProfile?: string;
   /**
@@ -1902,23 +1949,47 @@ export async function startVoiceTurn(
         conversation.toolsDisabledDepth++;
         frontDoorToolsSuppressed = true;
       }
+      // An escalated leg follows the conversation's own model: the front
+      // door hands off to the profile the caller's typed turns already run
+      // on, not to `callAgent`'s shipped default. Null keeps the ordinary
+      // call-site resolution.
+      const conversationProfile =
+        opts.routingLeg === "escalated"
+          ? conversationProfileForEscalation(conversation)
+          : null;
       // Resolved once here rather than inside the options literal below, so
       // the history scan happens once per leg. A front-door leg is skipped:
-      // its own call site already resolves to the same profile. The
-      // capability check comes before the scan because it is the cheaper of
-      // the two and it decides whether the pin is worth anything at all.
-      const carriesImage =
+      // its own call site already resolves to the same profile. A
+      // conversation profile whose model takes images needs no image pin
+      // either; one that does not yields to the image pin, since a model
+      // that rejects an image fails the whole leg. The judged profile is the
+      // concrete arm that serves this conversation, not a mix's name. The
+      // capability checks come before the scan because they are the cheaper
+      // of the two and they decide whether the pin is worth anything at all.
+      const needsImagePin =
         opts.routingLeg !== "front-door" &&
+        !(
+          conversationProfile != null &&
+          doesSupportVision(conversationProfile.modelProfile)
+        ) &&
         doesSupportVision(VOICE_IMAGE_PROFILE) &&
         conversationCarriesImage(conversation.getMessages());
-      if (carriesImage) {
+      if (needsImagePin) {
         log.info(
           { turnId, routingLeg: opts.routingLeg ?? null },
           "Voice leg carries an image; pinning the image-capable profile",
         );
+      } else if (conversationProfile != null) {
+        log.info(
+          { turnId, profile: conversationProfile.profile },
+          "Escalated voice leg pinned to the conversation's own profile",
+        );
       }
       const profilePin =
-        opts.overrideProfile ?? (carriesImage ? VOICE_IMAGE_PROFILE : null);
+        opts.overrideProfile ??
+        (needsImagePin
+          ? VOICE_IMAGE_PROFILE
+          : (conversationProfile?.profile ?? null));
       await conversation.runAgentLoop(persistedContent, messageId, {
         onEvent: (msg: AssistantEvent) => {
           if (msg.type === "assistant_turn_start") {
@@ -1995,15 +2066,17 @@ export async function startVoiceTurn(
         ...(isEscalationContinuation
           ? { messageKind: VOICE_ESCALATION_CONTINUATION_MESSAGE_KIND }
           : {}),
-        // Triage-and-escalate routing pins this turn to the fast front-door or
-        // strong escalation profile. `forceOverrideProfile` floats it above the
-        // callAgent call-site layers (callAgent is not `mainAgent`, so the
-        // override would otherwise sit below the call-site profile).
+        // Triage-and-escalate routing pins this turn to the fast front-door
+        // profile or to the conversation's own profile for the escalated
+        // leg. `forceOverrideProfile` floats it above the callAgent call-site
+        // layers (callAgent is not `mainAgent`, so the override would
+        // otherwise sit below the call-site profile).
         //
         // An explicit routing pin wins; failing that, a leg whose history
-        // carries an image is pinned to a profile whose model takes one. A
-        // front-door leg needs neither: its own call site already resolves
-        // there.
+        // carries an image is pinned to a profile whose model takes one;
+        // failing that, an escalated leg is pinned to the conversation's
+        // profile. A front-door leg needs none of these: its own call site
+        // already resolves there.
         ...(profilePin != null
           ? { overrideProfile: profilePin, forceOverrideProfile: true }
           : {}),

package/src/calls/voice-triage-escalate.ts

@@ -8,10 +8,10 @@
  *     caller is mid-thought — the leg is discarded and listening continues.
  *   - `[1]` ({@link ESCALATE_VERDICT_TOKEN}) followed by ONE short natural
  *     holding phrase: the turn is too tricky — the phrase is spoken (capped
- *     at a single sentence) while the turn re-runs on the call-site default
- *     profile, the model an un-routed voice turn would have used. Because
- *     the holding phrase is spoken, the caller never hears the stronger
- *     model's think-time as silence.
+ *     at a single sentence) while the turn re-runs on the conversation's
+ *     own profile, the model the caller's typed turns already run on.
+ *     Because the holding phrase is spoken, the caller never hears the
+ *     stronger model's think-time as silence.
  *   - Anything else: the output IS the answer, streamed straight to TTS
  *     (low first-token latency -> the caller hears audio fast).
  *
@@ -38,11 +38,13 @@ export { ESCALATE_VERDICT_TOKEN, HOLD_VERDICT_TOKEN };
 
 // The fast model fronting every turn is pinned by the `voiceFrontDoor` call
 // site (see config/call-site-defaults.ts) — no per-turn profile override.
-// The escalated leg likewise carries NO override: it runs on the ordinary
-// call-agent resolution, i.e. exactly the profile an un-routed voice turn
-// would use (balanced for a fresh workspace, or whatever the user pinned).
-// That guarantees an escalated answer is never weaker OR stronger than the
-// pre-routing behavior, and honors per-user profile choices.
+// The escalated leg is pinned by the bridge to the conversation's own
+// profile: the conversation's pinned profile if it has one, else the
+// workspace chat-model selection — the model the caller's typed turns in the
+// same conversation already run on (see `conversationProfileForEscalation`
+// in voice-session-bridge.ts). A conversation with no such selection keeps
+// the ordinary call-agent resolution. That honors the user's model choice
+// instead of dropping every hand-off onto the call site's shipped default.
 
 /**
  * Which leg of a triaged turn a `startVoiceTurn` call represents. Undefined

package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts

@@ -194,8 +194,9 @@ describe("live-voice triage-and-escalate routing", () => {
     const escalated = starter.mock.calls[1]?.[0];
     expect(frontDoor?.overrideProfile).toBeUndefined();
     expect(frontDoor?.routingLeg).toBe("front-door");
-    // The escalated leg runs on the ordinary call-agent resolution: no
-    // override either.
+    // The session passes no override either: the bridge itself pins the
+    // escalated leg to the conversation's own profile (covered in
+    // calls/__tests__/voice-session-bridge.test.ts).
     expect(escalated?.overrideProfile).toBeUndefined();
     expect(escalated?.routingLeg).toBe("escalated");
     expect(escalated?.content).toBe(ESCALATION_CONTINUATION_CONTENT);

package/src/live-voice/live-voice-session.ts

@@ -5853,11 +5853,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       }
     }
 
-    // No overrideProfile: the escalated leg runs on the call-site default —
-    // the exact profile an un-routed voice turn would use (see
-    // voice-triage-escalate.ts). The bridge phrase the caller just heard is
-    // handed along so the escalated continuation rule can quote it and ban
-    // a re-announcing echo ("Let me check…" twice in a row).
+    // No overrideProfile here: the bridge pins the escalated leg to the
+    // conversation's own profile, the model the caller's typed turns already
+    // run on (see voice-triage-escalate.ts). The bridge phrase the caller
+    // just heard is handed along so the escalated continuation rule can
+    // quote it and ban a re-announcing echo ("Let me check…" twice in a
+    // row).
     void this.startAssistantLeg(activeTurn, {
       content: ESCALATION_CONTINUATION_CONTENT,
       routingLeg: "escalated",