@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/calls/__tests__/call-funnel-telemetry.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Phone-call funnel rows: which ones the call store records and what they say.
+ *
+ * The recorder is mocked, as in `live-voice-session-telemetry.test.ts`: the
+ * outbox and its consent gate are covered by `telemetry-events-outbox.test.ts`.
+ * What these tests pin is the wiring that decides whether a row exists at all,
+ * which is where the measurement silently goes wrong: a call that never
+ * connects still has to count, a terminal status has to emit exactly one end
+ * row however many writes follow it, and a call nobody spoke on has to say so.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+const recordPhoneCallStarted = mock(() => null);
+const recordPhoneCallEnded = mock(() => null);
+
+// Spread the real module rather than replacing it: the call store pulls other
+// recorders out of it, and a bare replacement makes those imports fail.
+const onboardingEventsStore =
+  await import("../../onboarding/onboarding-events-store.js");
+mock.module("../../onboarding/onboarding-events-store.js", () => ({
+  ...onboardingEventsStore,
+  recordPhoneCallStarted,
+  recordPhoneCallEnded,
+}));
+
+import { getDb } from "../../persistence/db-connection.js";
+import { initializeDb } from "../../persistence/db-init.js";
+import { conversations } from "../../persistence/schema/index.js";
+
+await initializeDb();
+
+// Values load after `mock.module` so the store picks up the mocked recorder.
+const { createCallSession, recordCallEvent, updateCallSession } =
+  await import("../call-store.js");
+
+let conversationSeq = 0;
+
+function makeConversation(): string {
+  const id = `conv-funnel-${++conversationSeq}`;
+  const now = Date.now();
+  getDb()
+    .insert(conversations)
+    .values({
+      id,
+      title: `Test conversation ${id}`,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .run();
+  return id;
+}
+
+function startCall(opts?: {
+  direction?: "inbound" | "outbound";
+  callMode?: string;
+}): string {
+  const session = createCallSession({
+    conversationId: makeConversation(),
+    provider: "twilio",
+    direction: opts?.direction ?? "inbound",
+    fromNumber: "+15550100",
+    toNumber: "+15550101",
+    ...(opts?.callMode !== undefined ? { callMode: opts.callMode } : {}),
+  });
+  return session.id;
+}
+
+beforeEach(() => {
+  recordPhoneCallStarted.mockClear();
+  recordPhoneCallEnded.mockClear();
+});
+
+describe("call funnel start rows", () => {
+  test("an inbound call counts the moment its session row exists", () => {
+    // Before the provider has dialled and before any preflight can reject it:
+    // a call that never connects is the one the funnel most needs to count.
+    const callSessionId = startCall({ callMode: "normal" });
+
+    expect(recordPhoneCallStarted).toHaveBeenCalledTimes(1);
+    expect(recordPhoneCallStarted).toHaveBeenCalledWith({
+      callSessionId,
+      screen: "started_inbound:normal",
+    });
+  });
+
+  test("an outbound call is stamped by the direction it states", () => {
+    // Stated, not inferred: a verification or invite call dials out and
+    // carries no task, so task presence would misread as inbound.
+    startCall({ direction: "outbound", callMode: "verification" });
+
+    expect(recordPhoneCallStarted).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "started_outbound:verification" }),
+    );
+  });
+
+  test("an ordinary call stamps the normal mode it never writes", () => {
+    // createInboundVoiceSession and startCall leave callMode null; only the
+    // verification and invite flows set one.
+    startCall();
+
+    expect(recordPhoneCallStarted).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "started_inbound:normal" }),
+    );
+  });
+});
+
+describe("call funnel end rows", () => {
+  test("a completed call that took a turn reports no silence", () => {
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "in_progress" });
+    recordCallEvent(callSessionId, "call_connected");
+    recordCallEvent(callSessionId, "caller_spoke", {
+      transcript: "hello",
+      transport: "media-stream",
+    });
+
+    updateCallSession(callSessionId, { status: "completed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledTimes(1);
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith({
+      callSessionId,
+      screen: "ended_completed",
+      outcome: "completed",
+    });
+  });
+
+  test("a call that never connected is silent for that reason", () => {
+    const callSessionId = startCall({ callMode: "normal" });
+
+    updateCallSession(callSessionId, { status: "failed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith({
+      callSessionId,
+      screen: "ended_failed:silent_no_connect",
+      outcome: "failed",
+    });
+  });
+
+  test("a connected call nobody spoke on is silent on the line", () => {
+    // The distinction the rate lives on: the telephony leg worked and the
+    // conversation still never started.
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "in_progress" });
+    recordCallEvent(callSessionId, "call_connected");
+
+    updateCallSession(callSessionId, { status: "completed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "ended_completed:silent_no_turn" }),
+    );
+  });
+
+  test("a call where only digits were pressed took no turn", () => {
+    // DTMF digits ride `caller_spoke` too; only a transcript is a turn.
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "in_progress" });
+    recordCallEvent(callSessionId, "call_connected");
+    recordCallEvent(callSessionId, "caller_spoke", {
+      dtmfDigit: "4",
+      transport: "media-stream",
+    });
+
+    updateCallSession(callSessionId, { status: "completed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "ended_completed:silent_no_turn" }),
+    );
+  });
+
+  test("a cancelled call completed its course", () => {
+    // Only an error is a failure; a call cancelled before it connected ran
+    // exactly as far as it was asked to.
+    const callSessionId = startCall({ callMode: "normal" });
+
+    updateCallSession(callSessionId, { status: "cancelled" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ outcome: "completed" }),
+    );
+  });
+
+  test("only the first terminal write records an end", () => {
+    // Terminal states are immutable, so the rejected second write must not
+    // produce a second row to pair with the one start.
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "completed" });
+
+    updateCallSession(callSessionId, { status: "failed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledTimes(1);
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "ended_completed:silent_no_connect" }),
+    );
+  });
+
+  test("a non-status update records nothing", () => {
+    const callSessionId = startCall({ callMode: "normal" });
+
+    updateCallSession(callSessionId, { providerCallSid: "CA-test" });
+
+    expect(recordPhoneCallEnded).not.toHaveBeenCalled();
+  });
+});

package/src/calls/__tests__/voice-control-protocol.test.ts

@@ -10,6 +10,7 @@ import {
   type SessionControlRequest,
   stripInternalSpeechMarkers,
   TASK_STOP_MARKER,
+  TASK_UPDATE_SILENT_MARKER,
   terminalControlMarkerLength,
 } from "../voice-control-protocol.js";
 
@@ -227,3 +228,17 @@ describe("session control markers", () => {
     expect(terminalControlMarkerLength("The array [-1] sorts")).toBe(0);
   });
 });
+
+test("silent task-update marker is held across every split and removed from the transcript", () => {
+  for (let split = 1; split < TASK_UPDATE_SILENT_MARKER.length; split += 1) {
+    const emitted: string[] = [];
+    const flush = createControlMarkerHoldback((text) => emitted.push(text));
+    flush(TASK_UPDATE_SILENT_MARKER.slice(0, split));
+    flush(TASK_UPDATE_SILENT_MARKER, { force: true });
+    expect(emitted.join("")).toBe("");
+  }
+  expect(terminalControlMarkerLength(TASK_UPDATE_SILENT_MARKER)).toBe(
+    TASK_UPDATE_SILENT_MARKER.length,
+  );
+  expect(parseTerminalSessionControl(TASK_UPDATE_SILENT_MARKER)).toBeNull();
+});

package/src/calls/__tests__/voice-escalation-judge.test.ts

@@ -0,0 +1,209 @@
+import { describe, expect, test } from "bun:test";
+
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+} from "../../providers/types.js";
+import {
+  ESCALATION_JUDGE_THRESHOLD,
+  judgeEscalation,
+  recentConversationForJudge,
+  VOICE_ESCALATION_JUDGE_CALL_SITE,
+} from "../voice-escalation-judge.js";
+
+function text(role: "user" | "assistant", body: string): Message {
+  return { role, content: [{ type: "text", text: body }] };
+}
+
+function jevResponse(noul: number): ProviderResponse {
+  const answers = { answer: { type: "noul", noul } };
+  return {
+    content: [{ type: "text", text: JSON.stringify(answers) }],
+    model: "jev-latest",
+    usage: { inputTokens: 1, outputTokens: 1 },
+    stopReason: "end_turn",
+    rawResponse: { model: "jev-latest", answers },
+  };
+}
+
+function fakeProvider(
+  respond: (
+    messages: Message[],
+    options?: SendMessageOptions,
+  ) => Promise<ProviderResponse>,
+): Provider {
+  return {
+    name: "typesafe",
+    sendMessage: respond,
+  } as Provider;
+}
+
+describe("recentConversationForJudge", () => {
+  test("keeps the last spoken turns, drops injected blocks and the current utterance", () => {
+    const history: Message[] = [
+      text("user", "hey"),
+      text("assistant", "Want me to email Priya the numbers?"),
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "<turn_context>\nnoise\n</turn_context>" },
+          { type: "text", text: "Yeah go ahead." },
+        ],
+      },
+    ];
+
+    expect(recentConversationForJudge(history, "Yeah go ahead.")).toBe(
+      "caller: hey\nassistant: Want me to email Priya the numbers?",
+    );
+  });
+
+  test("skips tool traffic and caps the window", () => {
+    const history: Message[] = [
+      ...Array.from({ length: 10 }, (_, i) => text("user", `turn ${i}`)),
+      {
+        role: "assistant",
+        content: [
+          { type: "tool_use", id: "t1", name: "web_search", input: {} },
+        ],
+      },
+    ];
+
+    const lines = recentConversationForJudge(history, "next").split("\n");
+    expect(lines).toHaveLength(6);
+    expect(lines.at(-1)).toBe("caller: turn 9");
+  });
+});
+
+describe("judgeEscalation", () => {
+  const history = [text("assistant", "Hi, what can I do?")];
+
+  test("escalates at or above the threshold and asks through the judge call site", async () => {
+    let seen: { body: unknown; options?: SendMessageOptions } | null = null;
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "Text my mom I'm late.",
+      resolveProvider: async () =>
+        fakeProvider(async (messages, options) => {
+          const block = messages[0]!.content[0]!;
+          seen = {
+            body: JSON.parse(block.type === "text" ? block.text : ""),
+            options,
+          };
+          return jevResponse(ESCALATION_JUDGE_THRESHOLD);
+        }),
+    });
+
+    expect(judgement.escalate).toBe(true);
+    expect(judgement.outcome).toBe("escalate");
+    expect(judgement.noul).toBe(ESCALATION_JUDGE_THRESHOLD);
+    const request = seen as unknown as {
+      body: {
+        state: { caller_just_said: string; recent_conversation: string };
+        questions: { answer: { type: string } };
+      };
+      options: SendMessageOptions;
+    };
+    expect(request.body.state.caller_just_said).toBe("Text my mom I'm late.");
+    expect(request.body.state.recent_conversation).toBe(
+      "assistant: Hi, what can I do?",
+    );
+    expect(request.body.questions.answer.type).toBe("noul");
+    expect(
+      (request.options.config as { callSite?: string } | undefined)?.callSite,
+    ).toBe(VOICE_ESCALATION_JUDGE_CALL_SITE);
+  });
+
+  test("clears below the threshold", async () => {
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "Tell me a joke.",
+      resolveProvider: async () => fakeProvider(async () => jevResponse(0.05)),
+    });
+
+    expect(judgement).toMatchObject({ escalate: false, outcome: "clear" });
+  });
+
+  test("is unavailable without a TypeSafe provider", async () => {
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "Open Spotify.",
+      resolveProvider: async () => null,
+    });
+
+    expect(judgement).toMatchObject({
+      escalate: false,
+      outcome: "unavailable",
+    });
+  });
+
+  test("a provider error does not escalate", async () => {
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "Open Spotify.",
+      resolveProvider: async () =>
+        fakeProvider(async () => {
+          throw new Error("503");
+        }),
+    });
+
+    expect(judgement).toMatchObject({ escalate: false, outcome: "error" });
+  });
+
+  test("an unparseable answer does not escalate", async () => {
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "Open Spotify.",
+      resolveProvider: async () =>
+        fakeProvider(async () => ({
+          ...jevResponse(0.9),
+          content: [{ type: "text", text: "not json" }],
+          rawResponse: {},
+        })),
+    });
+
+    expect(judgement).toMatchObject({ escalate: false, outcome: "error" });
+  });
+
+  test("a provider that never answers times out without escalating", async () => {
+    let aborted = false;
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "Open Spotify.",
+      timeoutMs: 20,
+      resolveProvider: async () =>
+        fakeProvider(async (_messages, options) => {
+          options?.signal?.addEventListener("abort", () => {
+            aborted = true;
+          });
+          return new Promise<ProviderResponse>(() => {});
+        }),
+    });
+
+    expect(judgement).toMatchObject({ escalate: false, outcome: "timeout" });
+    expect(aborted).toBe(true);
+  });
+
+  test("an empty utterance is not judged", async () => {
+    let called = false;
+    const judgement = await judgeEscalation({
+      conversationId: "conv-1",
+      history,
+      utterance: "   ",
+      resolveProvider: async () => {
+        called = true;
+        return null;
+      },
+    });
+
+    expect(judgement.outcome).toBe("unavailable");
+    expect(called).toBe(false);
+  });
+});

package/src/calls/__tests__/voice-leg-coordinator.test.ts

@@ -29,6 +29,7 @@ function harness(opts?: {
   commit?: () => boolean;
   withProgress?: boolean;
   withHold?: boolean;
+  withOverrule?: boolean;
 }) {
   const recorded: Recorded = {
     events: [],
@@ -72,6 +73,9 @@ function harness(opts?: {
       recorded.events.push(`answer:${text}`);
     },
     abortLeg: () => recorded.events.push("abort"),
+    ...(opts?.withOverrule === false
+      ? {}
+      : { overruleLeg: () => recorded.events.push("overrule") }),
     speakBridge: (bridge) => {
       recorded.bridges.push(bridge);
       recorded.events.push(`speak:${bridge.spokenBridge}`);
@@ -227,3 +231,167 @@ describe("createFrontDoorLegCoordinator", () => {
     ]);
   });
 });
+
+/** A judge verdict the test settles by hand. */
+function deferredVerdict() {
+  let settle!: (escalate: boolean) => void;
+  const verdict = new Promise<boolean>((resolve) => {
+    settle = resolve;
+  });
+  return { verdict, settle };
+}
+
+const flush = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
+
+describe("createFrontDoorLegCoordinator with an escalation judge", () => {
+  test("an answer is held until the judge clears it, then streams", async () => {
+    const { coordinator, recorded } = harness();
+    const judge = deferredVerdict();
+    coordinator.attachEscalationJudge(judge.verdict);
+
+    coordinator.push("Yeah, ");
+    coordinator.push("sure.");
+    expect(recorded.answers).toEqual([]);
+    expect(coordinator.awaitingJudge).toBe(true);
+
+    judge.settle(false);
+    await flush();
+
+    expect(recorded.answers).toEqual(["Yeah, sure."]);
+    expect(coordinator.awaitingJudge).toBe(false);
+    coordinator.push(" More.");
+    expect(recorded.answers).toEqual(["Yeah, sure.", " More."]);
+    expect(coordinator.handedOff).toBe(false);
+  });
+
+  test("a judge escalation overrules a held answer and hands off with the canned bridge", async () => {
+    const { coordinator, recorded } = harness();
+    const judge = deferredVerdict();
+    coordinator.attachEscalationJudge(judge.verdict);
+
+    coordinator.push("I'm adding it to the draft now.");
+    judge.settle(true);
+    await flush();
+
+    expect(recorded.answers).toEqual([]);
+    expect(coordinator.handedOff).toBe(true);
+    expect(recorded.events).toEqual([
+      "progress.clear",
+      "overrule",
+      `speak:${FALLBACK_ESCALATION_BRIDGE}`,
+      "progress.floor",
+      "start-escalated",
+      "progress.arm",
+    ]);
+    expect(recorded.bridges[0]?.usesFallback).toBe(true);
+    expect(recorded.escalated).toEqual([
+      escalatedLegFor(FALLBACK_ESCALATION_BRIDGE),
+    ]);
+  });
+
+  test("a verdict in before the first answer word hands off on that word", async () => {
+    const { coordinator, recorded } = harness();
+    coordinator.attachEscalationJudge(Promise.resolve(true));
+    await flush();
+    expect(coordinator.handedOff).toBe(false);
+
+    coordinator.push("Sure, sending it.");
+
+    expect(recorded.answers).toEqual([]);
+    expect(coordinator.handedOff).toBe(true);
+    expect(recorded.events).toContain("overrule");
+  });
+
+  test("without an overrule hook the overruled leg is aborted", async () => {
+    const { coordinator, recorded } = harness({ withOverrule: false });
+    coordinator.attachEscalationJudge(Promise.resolve(true));
+    await flush();
+
+    coordinator.push("Sure.");
+
+    expect(recorded.events).toContain("abort");
+  });
+
+  test("the leg's own escalate verdict does not wait on the judge", () => {
+    const { coordinator, recorded } = harness();
+    coordinator.attachEscalationJudge(deferredVerdict().verdict);
+
+    coordinator.push("[1] Let me check that.");
+
+    expect(coordinator.handedOff).toBe(true);
+    expect(recorded.bridges[0]?.spokenBridge).toBe("Let me check that.");
+    expect(recorded.events).toContain("abort");
+    expect(recorded.events).not.toContain("overrule");
+  });
+
+  test("a hold verdict does not wait on the judge", () => {
+    const { coordinator, recorded } = harness({
+      holdEnabled: true,
+      withHold: true,
+    });
+    coordinator.attachEscalationJudge(deferredVerdict().verdict);
+
+    coordinator.push("[0]");
+
+    expect(recorded.holds).toBe(1);
+    expect(coordinator.awaitingJudge).toBe(false);
+  });
+
+  test("a judge attached after the answer started speaking is ignored", async () => {
+    const { coordinator, recorded } = harness();
+    coordinator.push("Canberra.");
+    coordinator.attachEscalationJudge(Promise.resolve(true));
+    await flush();
+
+    coordinator.push(" It's the capital.");
+
+    expect(recorded.answers).toEqual(["Canberra.", " It's the capital."]);
+    expect(coordinator.handedOff).toBe(false);
+  });
+
+  test("settled() resolves once a held answer is released", async () => {
+    const { coordinator, recorded } = harness();
+    const judge = deferredVerdict();
+    coordinator.attachEscalationJudge(judge.verdict);
+    coordinator.push("Done.");
+
+    let settled = false;
+    void coordinator.settled().then(() => {
+      settled = true;
+    });
+    await flush();
+    expect(settled).toBe(false);
+
+    judge.settle(false);
+    await flush();
+
+    expect(settled).toBe(true);
+    expect(recorded.answers).toEqual(["Done."]);
+    expect(coordinator.complete()).toBe(false);
+  });
+
+  test("a judge that rejects clears the held answer", async () => {
+    const { coordinator, recorded } = harness();
+    coordinator.attachEscalationJudge(Promise.reject(new Error("boom")));
+    coordinator.push("Hi there.");
+    await flush();
+
+    expect(recorded.answers).toEqual(["Hi there."]);
+  });
+
+  test("a dead turn never hands off on a judge escalation", async () => {
+    let live = true;
+    const { coordinator, recorded } = harness({ live: () => live });
+    const judge = deferredVerdict();
+    coordinator.attachEscalationJudge(judge.verdict);
+    coordinator.push("Sure.");
+
+    live = false;
+    judge.settle(true);
+    await flush();
+
+    expect(coordinator.handedOff).toBe(false);
+    expect(recorded.escalated).toEqual([]);
+    expect(recorded.answers).toEqual([]);
+  });
+});