@vellumai/assistant 0.12.0-dev.202609111617.f60422f → 0.12.0-dev.202609111724.a78dcda

package/ARCHITECTURE.md

@@ -612,7 +612,7 @@ Transcription mode is selected once per session in `media-stream-stt-session.ts`
 - **Streaming** (default): when `calls.voice.telephonyStreaming` is enabled and the `telephony` role resolves a streaming transcriber (`resolveStreamingTranscriber({ role: "telephony" })`), inbound audio is decoded (mu-law → PCM16, resampled 8 kHz → 16 kHz) and fed to the provider's realtime adapter. Replies trigger only on utterance-boundary finals (for Deepgram, `speech_final`/`UtteranceEnd`, never mid-sentence `is_final` segments), and barge-in fires from local energy VAD, never from transcriber partials.
 - **Batch fallback**: otherwise the session segments turns with the energy-based `MediaTurnDetector` and transcribes each completed turn via the same role's batch API. Both halves of a call read the `telephony` role, which is why a role names its consumer rather than a boundary.
 
-Every phone turn runs the same two-leg triage as live voice through `startVoiceTurn` (`src/calls/voice-session-bridge.ts`): `call-controller.ts` opens on a toolless front-door leg (`routingLeg: "front-door"`, the `voiceFrontDoor` call site) and reads its stream through the shared `createFrontDoorVerdictMachine`; an escalate verdict speaks the capped holding phrase (or the localized canned bridge) and continues on an escalated leg pinned to the conversation's own model. Phone has no partial transcripts, so the hold verdict is never taught and routing is escalate-only. The controller also passes the bridge's turn callbacks (tool activity is recorded as `tool_use_started` / `tool_use_completed` call events, persisted row ids ride the `assistant_spoke` event), `launchedAtMs` for dispatch timing, and a `voiceTelemetry` bag keyed by the call session with a `phone_inbound` / `phone_outbound` entry.
+Every phone turn runs the same two-leg triage as live voice through `startVoiceTurn` (`src/calls/voice-session-bridge.ts`): `call-controller.ts` opens on a toolless front-door leg (`routingLeg: "front-door"`, the `voiceFrontDoor` call site) and drives it through the shared `createFrontDoorLegCoordinator` (`src/calls/voice-leg-coordinator.ts`), which reads the stream through the verdict machine and sequences the hand-off (pause narration, abort the leg, resolve and speak the bridge, mark it as the floor holder, start the escalated leg pinned to the conversation's own model, re-arm narration); each driver supplies only a host for how text and the bridge are spoken, how a leg is started or aborted, and (live voice only) the speculative hold and commit. Phone has no partial transcripts, so the hold verdict is never taught and routing is escalate-only. The controller also passes the bridge's turn callbacks (tool activity is recorded as `tool_use_started` / `tool_use_completed` call events, persisted row ids ride the `assistant_spoke` event), `launchedAtMs` for dispatch timing, and a `voiceTelemetry` bag keyed by the call session with a `phone_inbound` / `phone_outbound` entry. Both drivers share the spoken progress narration cadence (`src/calls/voice-progress-cadence.ts`, tuned by `voice.frontModel.progress`): the cadence owns the tool-activity log, the triggers (an ops burst, a long operation completing, a full interval of audible silence with news, the `maxSilenceMs` heartbeat) and the generated or static phrase, while each driver supplies its own view of audible silence (live voice from its TTS queue and playback-tail estimate; the media-stream transport from `isPlaybackIdle()` and a running sum of sent frame durations) and how to speak a phrase.
 
 A credential preflight (`resolveTelephonyCredentialReadiness()` in `src/calls/telephony-credential-preflight.ts`) gates every call: it requires a credentialed, telephony-capable STT provider **and** a media-stream-playable TTS provider (the configured one or a credentialed playable fallback). Inbound calls that fail the preflight receive `<Say>` setup-required copy plus `<Hangup/>` instead of a doomed stream; outbound placement fails before dialing via `preflightVoiceIngress()` with the same user-facing message.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.12.0-dev.202609111617.f60422f",
+  "version": "0.12.0-dev.202609111724.a78dcda",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/config-schema.test.ts

@@ -998,6 +998,18 @@ describe("AssistantConfigSchema", () => {
         echoEmaHalfLifeMs: 400,
         echoDrainSlackMs: 300,
       },
+      flux: {
+        turnEnd: { enabled: true },
+        eotThreshold: 0.7,
+        eotTimeoutMs: 5000,
+      },
+      archiveAudio: false,
+    });
+  });
+
+  test("applies voice defaults", () => {
+    const result = AssistantConfigSchema.parse({});
+    expect(result.voice).toEqual({
       frontModel: {
         endpointDecisionTimeoutMs: 1200,
         endpointExtensionMs: 1500,
@@ -1012,12 +1024,6 @@ describe("AssistantConfigSchema", () => {
           generationTimeoutMs: 1500,
         },
       },
-      flux: {
-        turnEnd: { enabled: true },
-        eotThreshold: 0.7,
-        eotTimeoutMs: 5000,
-      },
-      archiveAudio: false,
     });
   });
 

package/src/calls/__tests__/call-controller.test.ts

@@ -225,6 +225,7 @@ import { resetDbForTesting } from "../../__tests__/db-test-helpers.js";
 import { createGuardianBinding } from "../../__tests__/helpers/create-guardian-binding.js";
 import { setConfig } from "../../__tests__/helpers/set-config.js";
 import { loadConfig } from "../../config/loader.js";
+import type { VoiceProgressConfig } from "../../config/schemas/voice.js";
 import { getMessages } from "../../persistence/conversation-crud.js";
 import { getDb } from "../../persistence/db-connection.js";
 import { initializeDb } from "../../persistence/db-init.js";
@@ -240,6 +241,7 @@ import {
   updateCallSession,
 } from "../call-store.js";
 import type { CallTransport } from "../call-transport.js";
+import type { VoiceProgressNarrator } from "../progress-narration.js";
 import { resolveCallTtsProvider } from "../resolve-call-tts-provider.js";
 import {
   ESCALATION_CONTINUATION_CONTENT,
@@ -412,6 +414,9 @@ function setupController(
     awaitPlaybackDrained?: () => Promise<void>;
     /** Synthesis-language resolver, as the media-stream server wires it. */
     resolveSynthesisLanguage?: () => string | undefined;
+    /** Progress narration seam; an explicit null keeps the call silent. */
+    progressNarrator?: VoiceProgressNarrator | null;
+    progressConfig?: VoiceProgressConfig;
   },
 ) {
   ensureConversation("conv-ctrl-test");
@@ -436,6 +441,11 @@ function setupController(
     assistantId: opts?.assistantId,
     trustContext: opts?.trustContext,
     resolveSynthesisLanguage: opts?.resolveSynthesisLanguage,
+    // Tests that do not exercise narration keep the default narrator out of
+    // the way: a null narrator never generates or speaks.
+    progressNarrator:
+      opts?.progressNarrator === undefined ? null : opts.progressNarrator,
+    progressConfig: opts?.progressConfig,
   });
   return { session, relay: transport, controller };
 }
@@ -4521,6 +4531,102 @@ describe("call-controller", () => {
       controller.destroy();
     });
 
+    test("progress narration speaks into the escalated leg's tool loop, audio-only", async () => {
+      const narrationConfig: VoiceProgressConfig = {
+        enabled: true,
+        opsThreshold: 1,
+        idleIntervalMs: 60_000,
+        maxSilenceMs: 60_000,
+        longOpMs: 15_000,
+        minGapMs: 10,
+        generationTimeoutMs: 1_500,
+      };
+      const narrator: VoiceProgressNarrator = {
+        generateProgressText: async (input) => {
+          expect(input.transcriptSoFar).toBe("What is on my calendar?");
+          expect(input.currentOp?.toolName).toBe("calendar_list");
+          return "Pulling up your calendar now.";
+        },
+      };
+      const { session, relay, controller } = setupController(undefined, {
+        progressNarrator: narrator,
+        progressConfig: narrationConfig,
+      });
+      const calls: LegOpts[] = [];
+      mockStartVoiceTurn.mockImplementation(async (opts: LegOpts) => {
+        const index = calls.length;
+        calls.push(opts);
+        if (index === 0) {
+          opts.onTextDelta("[1] One moment.");
+          opts.onComplete();
+          return { turnId: "run-front", abort: () => {} };
+        }
+        // The escalated leg runs a tool; narration must land while it runs.
+        // The bridge just spoken is a floor holder, so the tool starts after
+        // the (test-short) minimum gap.
+        await new Promise((r) => setTimeout(r, 25));
+        opts.callbacks?.tool_use_start?.("calendar_list", { toolUseId: "c1" });
+        await pollUntil(() =>
+          relay.sentTokens.some((t) =>
+            t.token.includes("Pulling up your calendar now."),
+          ),
+        );
+        opts.callbacks?.tool_result?.({
+          toolName: "calendar_list",
+          toolUseId: "c1",
+          resultPreview: "two events",
+        });
+        opts.onTextDelta("You have two events today.");
+        opts.onComplete();
+        return { turnId: "run-escalated", abort: () => {} };
+      });
+
+      await controller.handleCallerUtterance("What is on my calendar?");
+
+      const spoken = spokenText(relay);
+      expect(spoken.indexOf("One moment.")).toBeLessThan(
+        spoken.indexOf("Pulling up your calendar now."),
+      );
+      expect(spoken.indexOf("Pulling up your calendar now.")).toBeLessThan(
+        spoken.indexOf("You have two events today."),
+      );
+      // Narration is audio-only: the turn's recorded text is the model's.
+      expect(assistantSpokePayload(session.id).text).toBe(
+        "One moment. You have two events today.",
+      );
+
+      controller.destroy();
+    });
+
+    test("no narration without a narrator", async () => {
+      const { relay, controller } = setupController(undefined, {
+        progressNarrator: null,
+        progressConfig: {
+          enabled: true,
+          opsThreshold: 1,
+          idleIntervalMs: 60_000,
+          maxSilenceMs: 60_000,
+          longOpMs: 15_000,
+          minGapMs: 10,
+          generationTimeoutMs: 1_500,
+        },
+      });
+      scriptLegs([["Done."]], (opts) => {
+        opts.callbacks?.tool_use_start?.("web_search", { toolUseId: "t1" });
+        opts.callbacks?.tool_result?.({
+          toolName: "web_search",
+          toolUseId: "t1",
+          resultPreview: "ok",
+        });
+      });
+
+      await controller.handleCallerUtterance("Look it up");
+
+      expect(spokenText(relay)).toBe("Done.");
+
+      controller.destroy();
+    });
+
     test("an outbound call reports the outbound entry", async () => {
       const { calls } = scriptLegs([["Hello, this is Ava."]]);
       const { session, controller } = setupController("Book a table for two");

package/src/calls/__tests__/media-stream-output.test.ts

@@ -854,6 +854,38 @@ describe("MediaStreamOutput", () => {
       // Initially empty
       expect(output.getPlaybackQueueLength()).toBe(0);
     });
+
+    test("isPlaybackIdle tracks buffered text, the queue, and the sent-audio tail", async () => {
+      const samples = Array.from({ length: 4000 }, (_, i) =>
+        Math.round(Math.sin(i * 0.1) * 10000),
+      );
+      mockSynthesize.mockResolvedValue({
+        audio: makeWavBuffer(samples),
+        contentType: "audio/wav",
+      });
+
+      const { ws, sent } = createMockWs();
+      const output = makeOutput(ws, "stream-1");
+      expect(output.isPlaybackIdle()).toBe(true);
+      expect(output.playbackTailUntilMs()).toBe(0);
+
+      // Text buffered toward a sentence boundary is pending speech.
+      output.sendTextToken("Still thinking", false);
+      expect(output.isPlaybackIdle()).toBe(false);
+
+      output.sendTextToken(" about it.", true);
+      await drain(() => countEvents(sent, "mark") > 0);
+
+      // Frames left faster than real time: the tail estimate still covers
+      // the audio Twilio is playing out.
+      expect(output.playbackTailUntilMs()).toBeGreaterThan(Date.now());
+      expect(output.isPlaybackIdle()).toBe(false);
+
+      // A barge-in clear drops Twilio's buffer, so nothing is audible.
+      output.clearAudio();
+      expect(output.playbackTailUntilMs()).toBe(0);
+      expect(output.isPlaybackIdle()).toBe(true);
+    });
   });
 
   // ---------------------------------------------------------------------------

package/src/{live-voice → calls}/__tests__/progress-narration.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 
-import { LiveVoiceProgressConfigSchema } from "../../config/schemas/live-voice.js";
+import { VoiceProgressConfigSchema } from "../../config/schemas/voice.js";
 import type { Provider, ProviderResponse } from "../../providers/types.js";
 import {
   createVoiceProgressNarrator,
@@ -8,7 +8,7 @@ import {
   type VoiceProgressTextInput,
 } from "../progress-narration.js";
 
-const config = LiveVoiceProgressConfigSchema.parse({});
+const config = VoiceProgressConfigSchema.parse({});
 const progressInput: VoiceProgressTextInput = {
   transcriptSoFar: "compare flight prices for next month",
   completedOps: [
@@ -177,7 +177,7 @@ describe("createVoiceProgressNarrator", () => {
 
   test("bounds provider resolution by the configured timeout", async () => {
     const narrator = createVoiceProgressNarrator({
-      config: LiveVoiceProgressConfigSchema.parse({ generationTimeoutMs: 20 }),
+      config: VoiceProgressConfigSchema.parse({ generationTimeoutMs: 20 }),
       getProvider: () => new Promise<Provider | null>(() => {}),
     });
     const startedAt = Date.now();
@@ -187,7 +187,7 @@ describe("createVoiceProgressNarrator", () => {
 
   test("a caller abort settles promptly", async () => {
     const narrator = createVoiceProgressNarrator({
-      config: LiveVoiceProgressConfigSchema.parse({
+      config: VoiceProgressConfigSchema.parse({
         generationTimeoutMs: 60_000,
       }),
       getProvider: async () =>

package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts

@@ -8,7 +8,6 @@
 
 import { describe, expect, test } from "bun:test";
 
-import { BRIDGE_SENTENCE_END_REGEX } from "../../calls/voice-triage-escalate.js";
 import { DEEPGRAM_MULTI_LANGUAGE_CODES } from "../../providers/speech-to-text/deepgram.js";
 import {
   APPROVAL_PENDING_PHRASE,
@@ -18,6 +17,7 @@ import {
   PROGRESS_FALLBACK_PHRASES,
   PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
 } from "../progress-phrases.js";
+import { BRIDGE_SENTENCE_END_REGEX } from "../voice-triage-escalate.js";
 
 // Scripts without space-delimited words, where a word budget is
 // meaningless and length is asserted instead.

package/src/calls/__tests__/voice-leg-coordinator.test.ts

@@ -0,0 +1,229 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  createFrontDoorLegCoordinator,
+  type EscalatedLeg,
+  escalatedLegFor,
+  type FrontDoorLegHost,
+  type SpokenEscalationBridge,
+} from "../voice-leg-coordinator.js";
+import {
+  ESCALATION_CONTINUATION_CONTENT,
+  FALLBACK_ESCALATION_BRIDGE,
+  FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
+} from "../voice-triage-escalate.js";
+
+interface Recorded {
+  events: string[];
+  answers: string[];
+  bridges: SpokenEscalationBridge[];
+  escalated: EscalatedLeg[];
+  holds: number;
+  commits: number;
+}
+
+function harness(opts?: {
+  holdEnabled?: boolean;
+  live?: () => boolean;
+  language?: string;
+  commit?: () => boolean;
+  withProgress?: boolean;
+  withHold?: boolean;
+}) {
+  const recorded: Recorded = {
+    events: [],
+    answers: [],
+    bridges: [],
+    escalated: [],
+    holds: 0,
+    commits: 0,
+  };
+  const host: FrontDoorLegHost = {
+    isLive: opts?.live ?? (() => true),
+    language: () => opts?.language,
+    ...(opts?.withProgress === false
+      ? {}
+      : {
+          progress: {
+            clear: () => recorded.events.push("progress.clear"),
+            noteFloorHolder: () => recorded.events.push("progress.floor"),
+            arm: () => recorded.events.push("progress.arm"),
+          },
+        }),
+    ...(opts?.withHold
+      ? {
+          onHold: () => {
+            recorded.holds += 1;
+            recorded.events.push("hold");
+          },
+        }
+      : {}),
+    ...(opts?.commit
+      ? {
+          commit: () => {
+            recorded.commits += 1;
+            recorded.events.push("commit");
+            return opts.commit!();
+          },
+        }
+      : {}),
+    onAnswerText: (text) => {
+      recorded.answers.push(text);
+      recorded.events.push(`answer:${text}`);
+    },
+    abortLeg: () => recorded.events.push("abort"),
+    speakBridge: (bridge) => {
+      recorded.bridges.push(bridge);
+      recorded.events.push(`speak:${bridge.spokenBridge}`);
+    },
+    startEscalatedLeg: (leg) => {
+      recorded.escalated.push(leg);
+      recorded.events.push("start-escalated");
+    },
+  };
+  const coordinator = createFrontDoorLegCoordinator({
+    holdEnabled: opts?.holdEnabled ?? false,
+    host,
+  });
+  return { coordinator, recorded };
+}
+
+describe("escalatedLegFor", () => {
+  test("continues on the escalated leg with the spoken bridge quoted", () => {
+    expect(escalatedLegFor("One moment.")).toEqual({
+      content: ESCALATION_CONTINUATION_CONTENT,
+      routingLeg: "escalated",
+      spokenEscalationBridge: "One moment.",
+    });
+  });
+});
+
+describe("createFrontDoorLegCoordinator", () => {
+  test("an answer releases the held leading text, then streams", () => {
+    const { coordinator, recorded } = harness();
+    coordinator.push("[");
+    expect(recorded.answers).toEqual([]);
+    coordinator.push("END_CALL] Bye");
+    coordinator.push(" now.");
+
+    expect(recorded.answers).toEqual(["[END_CALL] Bye", " now."]);
+    expect(coordinator.complete()).toBe(false);
+    expect(coordinator.handedOff).toBe(false);
+    expect(recorded.escalated).toEqual([]);
+  });
+
+  test("an escalate verdict hands off in order: pause, abort, speak, floor, start, re-arm", () => {
+    const { coordinator, recorded } = harness();
+    coordinator.push("[1] Let me check");
+    expect(coordinator.handedOff).toBe(false);
+    coordinator.push(" that.");
+
+    expect(coordinator.handedOff).toBe(true);
+    expect(recorded.events).toEqual([
+      "progress.clear",
+      "abort",
+      "speak:Let me check that.",
+      "progress.floor",
+      "start-escalated",
+      "progress.arm",
+    ]);
+    expect(recorded.bridges[0]).toEqual({
+      spokenBridge: "Let me check that.",
+      usesFallback: false,
+    });
+    expect(recorded.escalated).toEqual([escalatedLegFor("Let me check that.")]);
+  });
+
+  test("after the hand-off the leg's stream and completion are ignored", () => {
+    const { coordinator, recorded } = harness();
+    coordinator.push("[1] One moment.");
+    coordinator.push(" Ignored past the cap.");
+    expect(coordinator.complete()).toBe(true);
+
+    expect(recorded.answers).toEqual([]);
+    expect(recorded.escalated.length).toBe(1);
+  });
+
+  test("a leg that stops mid-bridge hands off on completion with the canned bridge", () => {
+    const { coordinator, recorded } = harness({ language: "es" });
+    coordinator.push("[1]");
+    expect(coordinator.handedOff).toBe(false);
+
+    expect(coordinator.complete()).toBe(true);
+    expect(recorded.bridges[0]).toEqual({
+      spokenBridge: FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE.es,
+      usesFallback: true,
+    });
+    expect(recorded.escalated[0].spokenEscalationBridge).toBe(
+      FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE.es,
+    );
+  });
+
+  test("the canned bridge for a language the table lacks is English and says so", () => {
+    const { coordinator, recorded } = harness({ language: "ko" });
+    coordinator.push("[1]");
+    coordinator.complete();
+
+    expect(recorded.bridges[0]).toEqual({
+      spokenBridge: FALLBACK_ESCALATION_BRIDGE,
+      usesFallback: true,
+      language: "en",
+    });
+  });
+
+  test("a dead turn never hands off", () => {
+    let live = true;
+    const { coordinator, recorded } = harness({ live: () => live });
+    coordinator.push("[1] One");
+    live = false;
+    coordinator.push(" moment.");
+    expect(coordinator.handedOff).toBe(false);
+    expect(coordinator.complete()).toBe(false);
+
+    expect(recorded.escalated).toEqual([]);
+    expect(recorded.events).toEqual([]);
+  });
+
+  test("a hold verdict on a speculative leg discards the leg", () => {
+    const { coordinator, recorded } = harness({
+      holdEnabled: true,
+      withHold: true,
+    });
+    coordinator.push("[0]");
+
+    expect(recorded.holds).toBe(1);
+    expect(recorded.answers).toEqual([]);
+    expect(coordinator.handedOff).toBe(false);
+  });
+
+  test("a leg that does not teach the hold token reads [0] as an answer", () => {
+    const { coordinator, recorded } = harness({ withHold: true });
+    coordinator.push("[0] Sure.");
+
+    expect(recorded.holds).toBe(0);
+    expect(recorded.answers).toEqual(["[0] Sure."]);
+  });
+
+  test("answer and escalate verdicts commit a speculative turn first; a failed commit stops the verdict", () => {
+    const committed = harness({ commit: () => true });
+    committed.coordinator.push("Hello.");
+    expect(committed.recorded.events).toEqual(["commit", "answer:Hello."]);
+
+    const gone = harness({ commit: () => false });
+    gone.coordinator.push("[1] One moment.");
+    expect(gone.recorded.commits).toBe(1);
+    expect(gone.coordinator.handedOff).toBe(false);
+    expect(gone.recorded.escalated).toEqual([]);
+  });
+
+  test("works without a progress cadence", () => {
+    const { coordinator, recorded } = harness({ withProgress: false });
+    coordinator.push("[1] One moment.");
+
+    expect(recorded.events).toEqual([
+      "abort",
+      "speak:One moment.",
+      "start-escalated",
+    ]);
+  });
+});