@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/calls/typesafe-noul.ts

@@ -0,0 +1,156 @@
+/**
+ * One bounded yes/no question to TypeSafe's System One model (Jev), shared by
+ * the voice judges. A judge runs only when its call site resolves to the
+ * TypeSafe provider; every failure comes back as a non-answer so the caller
+ * keeps its default behavior.
+ */
+
+import type { LLMCallSite } from "../config/schemas/llm.js";
+import { type JevJsonValue, noulFromAnswer } from "../providers/jev/client.js";
+import { resolveConfiguredProvider } from "../providers/provider-send-message.js";
+import type { Provider, ProviderResponse } from "../providers/types.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("typesafe-noul");
+
+const TYPESAFE_PROVIDER_NAME = "typesafe";
+
+export type TypesafeNoulOutcome =
+  | "answered"
+  | "unavailable"
+  | "timeout"
+  | "error";
+
+export interface TypesafeNoulResult {
+  outcome: TypesafeNoulOutcome;
+  /** Calibrated P(yes), present when the outcome is "answered". */
+  noul?: number;
+  latencyMs: number;
+}
+
+/** The call site's provider when it resolves to TypeSafe, else null. */
+export async function resolveTypesafeProvider(
+  callSite: LLMCallSite,
+): Promise<Provider | null> {
+  const resolved = await resolveConfiguredProvider(callSite);
+  return resolved?.configuredProviderName === TYPESAFE_PROVIDER_NAME
+    ? resolved.provider
+    : null;
+}
+
+function answersFrom(
+  response: ProviderResponse,
+): Record<string, unknown> | null {
+  const raw = response.rawResponse;
+  if (
+    typeof raw === "object" &&
+    raw !== null &&
+    "answers" in raw &&
+    typeof raw.answers === "object" &&
+    raw.answers !== null
+  ) {
+    return raw.answers as Record<string, unknown>;
+  }
+  const block = response.content.find((b) => b.type === "text");
+  if (!block || block.type !== "text") {
+    return null;
+  }
+  try {
+    const parsed: unknown = JSON.parse(block.text);
+    return typeof parsed === "object" && parsed !== null
+      ? (parsed as Record<string, unknown>)
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Ask one noul question. Never rejects. The timeout is a hard race rather
+ * than trust in the provider honoring the abort, because callers hold user
+ * audio or background work on the answer.
+ */
+export async function askTypesafeNoul(args: {
+  callSite: LLMCallSite;
+  conversationId: string;
+  state: JevJsonValue;
+  instructions: string;
+  timeoutMs: number;
+  signal?: AbortSignal;
+  /** Defaults to {@link resolveTypesafeProvider} for `callSite`. */
+  resolveProvider?: () => Promise<Provider | null>;
+}): Promise<TypesafeNoulResult> {
+  const startedAt = Date.now();
+  const elapsed = () => Date.now() - startedAt;
+  const controller = new AbortController();
+  const signal = args.signal
+    ? AbortSignal.any([args.signal, controller.signal])
+    : controller.signal;
+
+  const ask = async (): Promise<TypesafeNoulResult> => {
+    try {
+      const provider = await (
+        args.resolveProvider ?? (() => resolveTypesafeProvider(args.callSite))
+      )();
+      if (!provider) {
+        return { outcome: "unavailable", latencyMs: elapsed() };
+      }
+      const response = await provider.sendMessage(
+        [
+          {
+            role: "user",
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify({
+                  state: args.state,
+                  questions: {
+                    answer: { type: "noul", instructions: args.instructions },
+                  },
+                }),
+              },
+            ],
+          },
+        ],
+        {
+          config: {
+            callSite: args.callSite,
+            conversationId: args.conversationId,
+            disableTurnStartCache: true,
+          },
+          signal,
+        },
+      );
+      const noul = noulFromAnswer(answersFrom(response)?.answer);
+      if (noul === undefined) {
+        log.warn(
+          { callSite: args.callSite, conversationId: args.conversationId },
+          "TypeSafe judge returned no usable answer",
+        );
+        return { outcome: "error", latencyMs: elapsed() };
+      }
+      return { outcome: "answered", noul, latencyMs: elapsed() };
+    } catch (err) {
+      if (!signal.aborted) {
+        log.warn(
+          { err, callSite: args.callSite, conversationId: args.conversationId },
+          "TypeSafe judge failed",
+        );
+      }
+      return { outcome: "error", latencyMs: elapsed() };
+    }
+  };
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timedOut = new Promise<TypesafeNoulResult>((resolve) => {
+    timer = setTimeout(() => {
+      controller.abort();
+      resolve({ outcome: "timeout", latencyMs: elapsed() });
+    }, args.timeoutMs);
+  });
+  try {
+    return await Promise.race([ask(), timedOut]);
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/calls/voice-control-protocol.ts

@@ -14,6 +14,7 @@ export const CALL_OPENING_ACK_MARKER = "[CALL_OPENING_ACK]";
 export const CALL_VERIFICATION_COMPLETE_MARKER = "[CALL_VERIFICATION_COMPLETE]";
 export const END_CALL_MARKER = "[END_CALL]";
 export const TASK_STOP_MARKER = "[TASK:STOP]";
+export const TASK_UPDATE_SILENT_MARKER = "[TASK_UPDATE:SILENT]";
 
 /**
  * Verdict tokens for the fast "front-door" model (triage-and-escalate voice
@@ -220,6 +221,7 @@ export function stripInternalSpeechMarkers(text: string): string {
     .replace(CALL_OPENING_ACK_MARKER_REGEX, "")
     .replace(END_CALL_MARKER_REGEX, "")
     .replace(TASK_STOP_MARKER_REGEX, "")
+    .replaceAll(TASK_UPDATE_SILENT_MARKER, "")
     .replace(HOLD_VERDICT_TOKEN_REGEX, "")
     .replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
     .replace(MINIMIZE_ROOM_MARKER_REGEX, "")
@@ -249,6 +251,7 @@ const CONTROL_MARKER_STRINGS = [
   "[CALL_OPENING_ACK]",
   "[END_CALL]",
   TASK_STOP_MARKER,
+  TASK_UPDATE_SILENT_MARKER,
   "[0]",
   "[1]",
   "[-1]",
@@ -435,6 +438,9 @@ export function parseTerminalSessionControl(
  */
 export function terminalControlMarkerLength(text: string): number {
   const trimmed = text.trimEnd();
+  if (trimmed.endsWith(TASK_UPDATE_SILENT_MARKER)) {
+    return TASK_UPDATE_SILENT_MARKER.length;
+  }
   if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
     return MINIMIZE_ROOM_MARKER.length;
   }

package/src/calls/voice-escalation-judge.ts

@@ -0,0 +1,145 @@
+/**
+ * Second opinion on the voice front door's escalate decision.
+ *
+ * The front-door leg is fast and toolless, and it decides in its leading
+ * tokens whether a turn needs the tool-capable escalated leg. Its failure mode
+ * is answering a turn it cannot complete ("yeah, I'll send that") so nothing
+ * ever happens. This judge asks TypeSafe's System One model one yes/no
+ * question about the same turn, in parallel with the front-door leg: does a
+ * correct reply need something the front door cannot do from here? A
+ * confident yes overrules a front-door answer and hands the turn off.
+ *
+ * The judge runs only when the `voiceEscalationJudge` call site resolves to
+ * the TypeSafe provider. Every other outcome (no such profile, provider
+ * error, timeout, unparseable answer) is a "clear" verdict, so the front
+ * door's own decision stands exactly as it would without the judge.
+ */
+
+import type { Message, Provider } from "../providers/types.js";
+import { safeStringSlice } from "../util/unicode.js";
+import { askTypesafeNoul } from "./typesafe-noul.js";
+
+export const VOICE_ESCALATION_JUDGE_CALL_SITE = "voiceEscalationJudge";
+
+/**
+ * Minimum P(yes) that overrules a front-door answer. Calibrated on a labeled
+ * set of voice turns: it caught nearly every action request, including bare
+ * confirmations of an earlier offer ("yeah, do it"), while clearing ordinary
+ * conversation, including follow-ups about work already done.
+ */
+export const ESCALATION_JUDGE_THRESHOLD = 0.7;
+
+/**
+ * Budget from the front-door dispatch. The judge's measured roundtrip sits
+ * well under the front-door model's time to first token, so within this
+ * budget the verdict is almost always in hand before the first answer word
+ * would have been spoken; past it the front door's decision stands.
+ */
+export const ESCALATION_JUDGE_TIMEOUT_MS = 800;
+
+/** Recent turns shown to the judge, oldest first. */
+const HISTORY_TURNS = 6;
+const HISTORY_TURN_MAX_CHARS = 600;
+
+export type EscalationJudgeOutcome =
+  | "escalate"
+  | "clear"
+  | "unavailable"
+  | "timeout"
+  | "error";
+
+export interface EscalationJudgement {
+  escalate: boolean;
+  outcome: EscalationJudgeOutcome;
+  /** Calibrated P(yes), when the judge answered. */
+  noul?: number;
+  latencyMs: number;
+}
+
+const ESCALATION_QUESTION =
+  "The voice assistant answering right now has NO tools and no access to the user's accounts, files, messages, calendar, screen, apps, the web, or saved memories. Would a correct reply to what the caller just said require doing something it cannot do from here: taking an action (sending, scheduling, creating, editing, changing, opening, calling), looking up live or current information, or retrieving a personal fact that is not already stated in the recent conversation? Answer yes if the caller is asking for, or agreeing to, any such action, even vaguely or via a short confirmation of an earlier offer. Answer no for conversation, opinions, general knowledge, and questions answerable from the recent conversation.";
+
+function textOf(message: Message): string {
+  return message.content
+    .map((block) => (block.type === "text" ? block.text : ""))
+    .filter(
+      (text) => text.trim().length > 0 && !text.trimStart().startsWith("<"),
+    )
+    .join(" ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * The judge's view of the conversation: the last few spoken turns as plain
+ * text, without injected context blocks, tool traffic, or the caller's
+ * current utterance (which the judge receives on its own).
+ */
+export function recentConversationForJudge(
+  history: readonly Message[],
+  utterance: string,
+): string {
+  const lines: string[] = [];
+  for (const message of history) {
+    const text = textOf(message);
+    if (text.length === 0) {
+      continue;
+    }
+    const speaker = message.role === "user" ? "caller" : "assistant";
+    lines.push(
+      `${speaker}: ${safeStringSlice(text, 0, HISTORY_TURN_MAX_CHARS)}`,
+    );
+  }
+  const current = `caller: ${utterance.replace(/\s+/g, " ").trim()}`;
+  if (lines.length > 0 && lines[lines.length - 1] === current) {
+    lines.pop();
+  }
+  return lines.slice(-HISTORY_TURNS).join("\n");
+}
+
+/**
+ * Judge whether a front-door turn must escalate. Never rejects: every failure
+ * is a non-escalating verdict, so the front door's decision stands.
+ */
+export async function judgeEscalation(args: {
+  conversationId: string;
+  history: readonly Message[];
+  utterance: string;
+  signal?: AbortSignal;
+  timeoutMs?: number;
+  /** Provider resolver, injectable for tests. */
+  resolveProvider?: () => Promise<Provider | null>;
+}): Promise<EscalationJudgement> {
+  const utterance = args.utterance.trim();
+  if (utterance.length === 0) {
+    return { escalate: false, outcome: "unavailable", latencyMs: 0 };
+  }
+  const result = await askTypesafeNoul({
+    callSite: VOICE_ESCALATION_JUDGE_CALL_SITE,
+    conversationId: args.conversationId,
+    state: {
+      recent_conversation:
+        recentConversationForJudge(args.history, utterance) ||
+        "(start of conversation)",
+      caller_just_said: utterance,
+    },
+    instructions: ESCALATION_QUESTION,
+    timeoutMs: args.timeoutMs ?? ESCALATION_JUDGE_TIMEOUT_MS,
+    ...(args.signal ? { signal: args.signal } : {}),
+    ...(args.resolveProvider ? { resolveProvider: args.resolveProvider } : {}),
+  });
+  if (result.outcome !== "answered" || result.noul === undefined) {
+    return {
+      escalate: false,
+      outcome: result.outcome === "answered" ? "error" : result.outcome,
+      latencyMs: result.latencyMs,
+    };
+  }
+  const escalate = result.noul >= ESCALATION_JUDGE_THRESHOLD;
+  return {
+    escalate,
+    outcome: escalate ? "escalate" : "clear",
+    noul: result.noul,
+    latencyMs: result.latencyMs,
+  };
+}

package/src/calls/voice-escalation-target.ts

@@ -0,0 +1,10 @@
+export type VoiceEscalationProfileSource =
+  | "conversation"
+  | "turn_override"
+  | "pre_model_hook"
+  | "call_site";
+
+export interface VoiceEscalationTarget {
+  profile: string;
+  source: VoiceEscalationProfileSource;
+}

package/src/calls/voice-leg-coordinator.ts

@@ -12,6 +12,12 @@
  * leg, resolve and speak the bridge, mark it as the floor holder, start the
  * escalated leg, re-arm narration. A leg that was cancelled never hands off.
  *
+ * An optional escalation judge (see `voice-escalation-judge.ts`) runs beside
+ * the leg. While its verdict is pending, an answer is held rather than
+ * spoken; a judge that says the turn needs the escalated leg overrules the
+ * answer and hands off with the canned bridge. Hold and escalate verdicts
+ * from the leg itself never wait on the judge.
+ *
  * Everything transport-specific comes in through {@link FrontDoorLegHost}:
  * how answer text and the bridge are spoken, how a leg is started or
  * aborted, and the speculative hold and commit that only live voice has.
@@ -90,6 +96,13 @@ export interface FrontDoorLegHost {
    * bridge cap adds no latency before the escalated leg starts.
    */
   abortLeg(): void;
+  /**
+   * Abort a leg whose answer the escalation judge overruled. Its persisted
+   * row holds an answer nobody heard, so the driver drops it rather than
+   * leaving it in the history the escalated leg reads. Falls back to
+   * {@link abortLeg} when absent.
+   */
+  overruleLeg?(): void;
   /**
    * Speak the bridge so the strong-model call has no dead air. The model's
    * own bridge is real assistant speech; the canned fallback is audio-only,
@@ -103,6 +116,21 @@ export interface FrontDoorLegHost {
 export interface FrontDoorLegCoordinator {
   /** Whether this leg handed the turn off to the escalated leg. */
   readonly handedOff: boolean;
+  /**
+   * True while answer text is held for a pending judge verdict. A driver
+   * whose leg completes in this state waits on {@link settled} before
+   * calling {@link complete}, so the held answer is spoken or overruled
+   * before the turn finishes.
+   */
+  readonly awaitingJudge: boolean;
+  /** Resolves once no answer text is held for the judge. */
+  settled(): Promise<void>;
+  /**
+   * Attach the escalation judge's verdict for this leg. `true` means the
+   * turn needs the escalated leg. The promise must never reject, and must
+   * settle within a bounded time: a held answer waits on it.
+   */
+  attachEscalationJudge(verdict: Promise<boolean>): void;
   /** Feed one raw delta of the front-door leg's stream. */
   push(text: string): void;
   /**
@@ -125,14 +153,35 @@ export function createFrontDoorLegCoordinator(options: {
   const { host } = options;
   const verdict = createFrontDoorVerdictMachine(options.holdEnabled);
   let handedOff = false;
+  // "none": no judge attached. "pending": attached, verdict not in yet.
+  let judge: "none" | "pending" | "escalate" | "clear" = "none";
+  // Answer text released by the verdict machine while the judge is pending.
+  // Null until the leg's verdict classified as an answer.
+  let heldAnswer: string | null = null;
+  let settleWaiters: Array<() => void> = [];
+  // Whether any answer text reached the host. A judge attached after that
+  // is ignored: speech already started cannot be overruled.
+  let answerReleased = false;
+
+  const releaseAnswer = (text: string): void => {
+    answerReleased = true;
+    host.onAnswerText(text);
+  };
 
-  const handOff = (cappedBridge: string): void => {
+  const handOff = (
+    cappedBridge: string,
+    opts: { overruled?: boolean } = {},
+  ): void => {
     if (handedOff || !host.isLive()) {
       return;
     }
     handedOff = true;
     host.progress?.clear();
-    host.abortLeg();
+    if (opts.overruled === true && host.overruleLeg !== undefined) {
+      host.overruleLeg();
+    } else {
+      host.abortLeg();
+    }
     const bridge = resolveSpokenEscalationBridge(cappedBridge, host.language());
     host.speakBridge(bridge);
     // The bridge is the turn's spoken acknowledgement: narration keeps its
@@ -145,10 +194,79 @@ export function createFrontDoorLegCoordinator(options: {
     host.progress?.arm();
   };
 
+  const releaseSettled = (): void => {
+    const waiters = settleWaiters;
+    settleWaiters = [];
+    for (const resolve of waiters) {
+      resolve();
+    }
+  };
+
+  // Act on held answer text once the judge has spoken: overrule it, or speak
+  // it as the leg would have without a judge.
+  const resolveHeldAnswer = (): void => {
+    if (heldAnswer === null || judge === "pending") {
+      return;
+    }
+    const text = heldAnswer;
+    heldAnswer = null;
+    if (judge === "escalate") {
+      handOff("", { overruled: true });
+    } else if (!handedOff && text.length > 0) {
+      releaseAnswer(text);
+    }
+    releaseSettled();
+  };
+
+  const onAnswer = (text: string): void => {
+    if (heldAnswer !== null) {
+      heldAnswer += text;
+      return;
+    }
+    if (judge === "pending") {
+      heldAnswer = text;
+      return;
+    }
+    if (judge === "escalate") {
+      // The verdict landed before the leg's first answer word.
+      heldAnswer = text;
+      resolveHeldAnswer();
+      return;
+    }
+    releaseAnswer(text);
+  };
+
   return {
     get handedOff() {
       return handedOff;
     },
+    get awaitingJudge() {
+      return heldAnswer !== null && judge === "pending";
+    },
+    settled() {
+      if (heldAnswer === null || judge !== "pending") {
+        return Promise.resolve();
+      }
+      return new Promise<void>((resolve) => {
+        settleWaiters.push(resolve);
+      });
+    },
+    attachEscalationJudge(verdict) {
+      if (judge !== "none" || answerReleased || handedOff) {
+        return;
+      }
+      judge = "pending";
+      void verdict.then(
+        (escalate) => {
+          judge = escalate ? "escalate" : "clear";
+          resolveHeldAnswer();
+        },
+        () => {
+          judge = "clear";
+          resolveHeldAnswer();
+        },
+      );
+    },
     push(text) {
       if (handedOff) {
         return;
@@ -166,7 +284,7 @@ export function createFrontDoorLegCoordinator(options: {
         return;
       }
       if (step.kind === "answer") {
-        host.onAnswerText(step.text);
+        onAnswer(step.text);
         return;
       }
       if (