@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/live-voice/camera-mode-session.ts

@@ -0,0 +1,277 @@
+import type { ModeSession, ModeSessionMode } from "../api/mode-session.js";
+import type {
+  ConversationModeSessionCoordinator,
+  ModeSessionSourceHandle,
+  ModeSessionTerminalDisposition,
+} from "../daemon/conversation-mode-session.js";
+import type { LiveVoiceSightSource } from "./protocol.js";
+
+type ModeSessionCoordinator = Pick<
+  ConversationModeSessionCoordinator,
+  | "activateSource"
+  | "claimTurn"
+  | "recordActivity"
+  | "releaseTurn"
+  | "retireSource"
+  | "trackPersistedRow"
+>;
+
+interface CameraRun {
+  readonly epoch: number;
+  readonly source: LiveVoiceSightSource;
+  readonly handle: ModeSessionSourceHandle;
+  readonly syntheticTurnId: string;
+  pendingFrames: number;
+  retired: boolean;
+}
+
+export interface CameraFrameToken {
+  readonly owner: ModeSessionSourceHandle;
+  readonly epoch: number;
+  readonly activation: number;
+}
+
+function modeFor(source: LiveVoiceSightSource): ModeSessionMode {
+  return source === "ambient" ? "ambient" : "live_vision";
+}
+
+/** Maps one voice session's negotiated camera epochs onto durable sessions. */
+export class CameraModeSessionProducer {
+  readonly #coordinator: ModeSessionCoordinator;
+  readonly #sourceId: string;
+  readonly #runs = new Map<number, CameraRun>();
+  readonly #pendingFrameTokens = new Set<CameraFrameToken>();
+  #activeRun?: CameraRun;
+  #highestEpoch = 0;
+
+  constructor(coordinator: ModeSessionCoordinator, voiceSessionId: string) {
+    this.#coordinator = coordinator;
+    this.#sourceId = `live-voice-camera:${voiceSessionId}`;
+  }
+
+  start(
+    epoch: number,
+    source: LiveVoiceSightSource = "live",
+    at = Date.now(),
+  ): ModeSessionSourceHandle | undefined {
+    const known = this.#runs.get(epoch);
+    if (known) {
+      return !known.retired && known.source === source
+        ? known.handle
+        : undefined;
+    }
+    if (epoch <= this.#highestEpoch) {
+      return undefined;
+    }
+
+    if (this.#activeRun) {
+      this.#retire(this.#activeRun, {
+        status: "interrupted",
+        endReason: "camera_source_replaced",
+      });
+    }
+
+    const handle = this.#coordinator.activateSource({
+      sourceId: this.#sourceId,
+      generation: epoch,
+      mode: modeFor(source),
+      sourceStartedAt: at,
+      lifetime: "source",
+    });
+    if (!handle) {
+      return undefined;
+    }
+
+    const syntheticTurnId = `${this.#sourceId}:${epoch}:${handle.activation}`;
+    let owner: ModeSession | undefined;
+    try {
+      owner = this.#coordinator.claimTurn(syntheticTurnId, handle, at);
+    } catch (err) {
+      const rollbackError = this.#rollbackStart(handle, syntheticTurnId);
+      if (rollbackError) {
+        throw new AggregateError(
+          [err, rollbackError],
+          "Camera session claim and rollback both failed",
+        );
+      }
+      throw err;
+    }
+    if (!owner || owner.id !== handle.id) {
+      const rollbackError = this.#rollbackStart(handle, syntheticTurnId);
+      if (rollbackError) {
+        throw rollbackError;
+      }
+      return undefined;
+    }
+
+    const run: CameraRun = {
+      epoch,
+      source,
+      handle,
+      syntheticTurnId,
+      pendingFrames: 0,
+      retired: false,
+    };
+    this.#runs.set(epoch, run);
+    this.#highestEpoch = epoch;
+    this.#activeRun = run;
+    return handle;
+  }
+
+  #rollbackStart(
+    handle: ModeSessionSourceHandle,
+    syntheticTurnId: string,
+  ): unknown {
+    let rollbackError: unknown;
+    try {
+      this.#coordinator.retireSource(handle, {
+        status: "interrupted",
+        endReason: "camera_start_refused",
+      });
+    } catch (err) {
+      rollbackError = err;
+    }
+    try {
+      this.#coordinator.releaseTurn(syntheticTurnId);
+    } catch (err) {
+      rollbackError ??= err;
+    }
+    return rollbackError;
+  }
+
+  captureTurn(): ModeSessionSourceHandle | undefined {
+    return this.#activeRun?.handle;
+  }
+
+  holdDelivery(
+    turnId: string,
+    at = Date.now(),
+  ): ModeSessionSourceHandle | undefined {
+    const handle = this.captureTurn();
+    if (!handle) {
+      return undefined;
+    }
+    const owner = this.#coordinator.claimTurn(turnId, handle, at);
+    return owner?.id === handle.id ? handle : undefined;
+  }
+
+  releaseDelivery(turnId: string): void {
+    this.#coordinator.releaseTurn(turnId);
+  }
+
+  beginFrame(
+    epoch: number,
+    source: LiveVoiceSightSource | undefined,
+  ): CameraFrameToken | undefined {
+    const run = this.#activeRun;
+    if (
+      !run ||
+      run.retired ||
+      run.epoch !== epoch ||
+      (source !== undefined && source !== run.source)
+    ) {
+      return undefined;
+    }
+    run.pendingFrames += 1;
+    const token = {
+      owner: run.handle,
+      epoch: run.epoch,
+      activation: run.handle.activation,
+    };
+    this.#pendingFrameTokens.add(token);
+    return token;
+  }
+
+  finishFrame(
+    token: CameraFrameToken | undefined,
+    persisted?: { messageId: string; at: number },
+  ): void {
+    if (!token || !this.#pendingFrameTokens.delete(token)) {
+      return;
+    }
+    const run = this.#runs.get(token.epoch);
+    if (!run || run.handle.activation !== token.activation) {
+      return;
+    }
+    try {
+      if (
+        persisted &&
+        this.#coordinator.recordActivity(run.syntheticTurnId, persisted.at)
+      ) {
+        this.#coordinator.trackPersistedRow(
+          run.syntheticTurnId,
+          persisted.messageId,
+          persisted.at,
+        );
+        this.#coordinator.recordActivity(run.syntheticTurnId, persisted.at);
+      }
+    } finally {
+      run.pendingFrames = Math.max(0, run.pendingFrames - 1);
+      this.#releaseIfSettled(run);
+    }
+  }
+
+  end(
+    epoch: number,
+    disposition: ModeSessionTerminalDisposition = {
+      status: "completed",
+      endReason: "camera_stopped",
+    },
+  ): boolean {
+    const run = this.#runs.get(epoch);
+    if (!run || run.retired) {
+      return false;
+    }
+    return this.#retire(run, disposition);
+  }
+
+  close(disposition: ModeSessionTerminalDisposition): boolean {
+    const run = this.#activeRun;
+    return run ? this.#retire(run, disposition) : false;
+  }
+
+  #retire(
+    run: CameraRun,
+    disposition: ModeSessionTerminalDisposition,
+  ): boolean {
+    if (run.retired) {
+      return false;
+    }
+    let retired = false;
+    let retirementError: unknown;
+    try {
+      retired = this.#coordinator.retireSource(run.handle, disposition);
+    } catch (err) {
+      retirementError = err;
+    }
+    run.retired = true;
+    if (this.#activeRun === run) {
+      this.#activeRun = undefined;
+    }
+    let releaseError: unknown;
+    try {
+      this.#releaseIfSettled(run);
+    } catch (err) {
+      releaseError = err;
+    }
+    if (retirementError && releaseError) {
+      throw new AggregateError(
+        [retirementError, releaseError],
+        "Camera session retirement and turn release both failed",
+      );
+    }
+    if (retirementError) {
+      throw retirementError;
+    }
+    if (releaseError) {
+      throw releaseError;
+    }
+    return retired;
+  }
+
+  #releaseIfSettled(run: CameraRun): void {
+    if (run.retired && run.pendingFrames === 0) {
+      this.#coordinator.releaseTurn(run.syntheticTurnId);
+    }
+  }
+}

package/src/live-voice/continuation-judge.ts

@@ -0,0 +1,123 @@
+/**
+ * Decides whether a barged-in request should keep running as a background
+ * continuation, from what the caller said when they interrupted.
+ *
+ * A barge-in hands the interrupted turn's work to a background subagent so
+ * it is not lost. That is right when the caller moved on to something else
+ * ("oh, also, what's the weather?") or only acknowledged it ("okay, thanks"),
+ * and wrong when the interruption cancels or changes the request ("actually,
+ * just point it out for me"): the continuation then does work the caller
+ * no longer wants, while the foreground turn handles the changed request.
+ *
+ * The judge runs only when the `voiceContinuationJudge` call site resolves to
+ * the TypeSafe provider. Without one, the session never waits on it. Every
+ * failure keeps the continuation, which is the behavior without a judge.
+ */
+
+import {
+  askTypesafeNoul,
+  resolveTypesafeProvider,
+} from "../calls/typesafe-noul.js";
+import type { Provider } from "../providers/types.js";
+
+export const VOICE_CONTINUATION_JUDGE_CALL_SITE = "voiceContinuationJudge";
+
+/**
+ * Minimum P(yes) that keeps the continuation. Calibrated on labeled
+ * interruptions: cancels, corrections, and repeats scored 0.05 to 0.21;
+ * unrelated requests, acknowledgements, and progress questions scored 0.56
+ * to 0.89.
+ */
+export const CONTINUATION_JUDGE_KEEP_THRESHOLD = 0.4;
+
+export const CONTINUATION_JUDGE_TIMEOUT_MS = 1_500;
+
+const KEEP_QUESTION =
+  "The voice assistant was working on the caller's earlier request when the caller interrupted it. Should the assistant still finish that earlier request in the background, exactly as originally asked? Answer yes if the caller's new words are unrelated to it (a separate question, request, or remark), an acknowledgement, or a question about its progress. Answer no if the new words cancel it, change the caller's mind about it, correct or change what should be done, ask for it differently, or simply repeat it.";
+
+export type ContinuationJudgeOutcome =
+  | "keep"
+  | "drop"
+  | "unavailable"
+  | "no_interruption"
+  | "timeout"
+  | "error";
+
+export interface ContinuationJudgement {
+  keep: boolean;
+  outcome: ContinuationJudgeOutcome;
+  noul?: number;
+  latencyMs: number;
+}
+
+/**
+ * Judges one barge-in. `interruption` resolves with the caller's interrupting
+ * words once the next turn dispatches, or null when none arrived in time;
+ * the judge awaits it only when a TypeSafe profile is configured. Never
+ * rejects.
+ */
+export type LiveVoiceContinuationJudge = (args: {
+  parentConversationId: string;
+  interruptedRequest: string;
+  interruption: Promise<string | null>;
+  signal: AbortSignal;
+}) => Promise<ContinuationJudgement>;
+
+export function createContinuationJudge(options?: {
+  /** Provider resolver, injectable for tests. */
+  resolveProvider?: () => Promise<Provider | null>;
+  timeoutMs?: number;
+}): LiveVoiceContinuationJudge {
+  const resolveProvider =
+    options?.resolveProvider ??
+    (() => resolveTypesafeProvider(VOICE_CONTINUATION_JUDGE_CALL_SITE));
+  return async (args) => {
+    const startedAt = Date.now();
+    const kept = (
+      outcome: ContinuationJudgeOutcome,
+      noul?: number,
+    ): ContinuationJudgement => ({
+      keep: true,
+      outcome,
+      ...(noul !== undefined ? { noul } : {}),
+      latencyMs: Date.now() - startedAt,
+    });
+    let provider: Provider | null;
+    try {
+      provider = await resolveProvider();
+    } catch {
+      return kept("error");
+    }
+    if (!provider || args.interruptedRequest.trim().length === 0) {
+      return kept("unavailable");
+    }
+    const interruption = (await args.interruption)?.trim() ?? "";
+    if (interruption.length === 0) {
+      return kept("no_interruption");
+    }
+    const result = await askTypesafeNoul({
+      callSite: VOICE_CONTINUATION_JUDGE_CALL_SITE,
+      conversationId: args.parentConversationId,
+      state: {
+        interrupted_request: args.interruptedRequest.trim(),
+        caller_then_said: interruption,
+      },
+      instructions: KEEP_QUESTION,
+      timeoutMs: options?.timeoutMs ?? CONTINUATION_JUDGE_TIMEOUT_MS,
+      signal: args.signal,
+      resolveProvider: async () => provider,
+    });
+    if (result.outcome !== "answered" || result.noul === undefined) {
+      return kept(result.outcome === "answered" ? "error" : result.outcome);
+    }
+    if (result.noul >= CONTINUATION_JUDGE_KEEP_THRESHOLD) {
+      return kept("keep", result.noul);
+    }
+    return {
+      keep: false,
+      outcome: "drop",
+      noul: result.noul,
+      latencyMs: Date.now() - startedAt,
+    };
+  };
+}

package/src/live-voice/live-activity-reporter.ts

@@ -33,6 +33,9 @@ import type { LiveVoiceServerFramePayload } from "./protocol.js";
 
 const log = getLogger("live-activity-reporter");
 
+/** A stale phase is preferable to letting one request pin every later phase. */
+const LIVE_ACTIVITY_DISPATCH_TIMEOUT_MS = 5_000;
+
 /** The phases an iOS Live Activity can render. Mirrors the client's union. */
 export type LiveActivityPhase =
   | "connecting"
@@ -42,6 +45,12 @@ export type LiveActivityPhase =
   | "speaking"
   | "ending";
 
+interface PendingDispatch {
+  phase: LiveActivityPhase;
+  event: "update" | "end";
+  detail: string;
+}
+
 /**
  * The phase a frame puts the session into, or `null` for frames that do not
  * move it.
@@ -88,6 +97,10 @@ export function phaseForFrame(
       // Unconditional, unlike `stt_final`: the session sends this frame
       // precisely when it has committed to a turn.
       return "thinking";
+    case "activity":
+      // Keep model-routing detail inside the conversation, while ensuring a
+      // silent escalation leaves any preceding speaking phase behind.
+      return frame.kind === "escalation" ? "thinking" : null;
     case "tts_audio":
       return "speaking";
     case "tts_done":
@@ -111,6 +124,11 @@ export class LiveActivityReporter {
   private lastPhase: LiveActivityPhase | null = null;
   private lastDetail = "";
   private ended = false;
+  private dispatchInFlight = false;
+  private pendingDispatch: PendingDispatch | null = null;
+  private dispatchIdleWaiters: Array<() => void> = [];
+  protected readonly dispatchTimeoutMs: number =
+    LIVE_ACTIVITY_DISPATCH_TIMEOUT_MS;
 
   constructor(private readonly conversationId: string) {}
 
@@ -128,10 +146,14 @@ export class LiveActivityReporter {
       return;
     }
     const phase = phaseForFrame(frame, this.lastPhase);
-    // The session sends this frame only on a change it wants surfaced, and
-    // sends an empty label when the turn stops working, so it is taken
-    // verbatim rather than derived.
-    const detail = frame.type === "activity" ? frame.label : this.lastDetail;
+    // Tool and approval activity is server-owned wording. Escalation is an
+    // in-conversation status, so keep it out of the system-level Live Activity.
+    const detail =
+      frame.type === "activity"
+        ? frame.kind === "escalation"
+          ? ""
+          : frame.label
+        : this.lastDetail;
     const phaseMoved = phase !== null && phase !== this.lastPhase;
     if (!phaseMoved && detail === this.lastDetail) {
       return;
@@ -147,7 +169,19 @@ export class LiveActivityReporter {
     if (this.lastPhase === null) {
       return;
     }
-    void this.dispatch(this.lastPhase, "update", this.lastDetail);
+    this.enqueueDispatch(this.lastPhase, "update", this.lastDetail);
+  }
+
+  /**
+   * Move the system surface past escalation bridge speech without replacing a
+   * tool or approval line that became active while that speech was draining.
+   */
+  restoreThinkingPhase(): void {
+    if (this.ended || this.lastPhase === "thinking") {
+      return;
+    }
+    this.lastPhase = "thinking";
+    this.enqueueDispatch("thinking", "update", this.lastDetail);
   }
 
   /**
@@ -166,7 +200,80 @@ export class LiveActivityReporter {
     // No detail on the way out: whatever the turn was doing, it is not doing
     // it any more, and this state is the one that lingers on the Lock Screen
     // through the dismissal window.
-    void this.dispatch("ending", "end", "");
+    this.enqueueDispatch("ending", "end", "");
+  }
+
+  /**
+   * Preserve frame order across asynchronous platform requests without making
+   * the session wait for them. A later phase must never arrive before an older
+   * request finishes and then be overwritten by that older content state.
+   */
+  private enqueueDispatch(
+    phase: LiveActivityPhase,
+    event: "update" | "end",
+    detail: string,
+  ): void {
+    const pending = { phase, event, detail } satisfies PendingDispatch;
+    if (this.dispatchInFlight) {
+      // ActivityKit renders snapshots, so only the newest state behind the
+      // current request matters. In particular, `end` replaces every queued
+      // update and is always the next request sent.
+      this.pendingDispatch = pending;
+      return;
+    }
+    this.startDispatch(pending);
+  }
+
+  private startDispatch(pending: PendingDispatch): void {
+    this.dispatchInFlight = true;
+    const signal = AbortSignal.timeout(this.dispatchTimeoutMs);
+    const deadline = new Promise<void>((resolve) => {
+      if (signal.aborted) {
+        resolve();
+        return;
+      }
+      signal.addEventListener("abort", () => resolve(), { once: true });
+    });
+    // Race the whole operation, including platform-client and credential
+    // resolution. The same signal reaches `fetch`, so a timed-out operation
+    // that resolves credentials later cannot send its stale snapshot.
+    void Promise.race([
+      this.dispatch(pending.phase, pending.event, pending.detail, signal),
+      deadline,
+    ])
+      .catch((err: unknown) => {
+        // `dispatch` contains its own best-effort error boundary. Keep this
+        // guard for subclasses and future implementations so one rejection
+        // cannot poison every later phase in the queue.
+        log.debug(
+          { err, phase: pending.phase, event: pending.event },
+          "Live Activity dispatch queue failed",
+        );
+      })
+      .finally(() => {
+        const next = this.pendingDispatch;
+        this.pendingDispatch = null;
+        if (next !== null) {
+          this.startDispatch(next);
+          return;
+        }
+        this.dispatchInFlight = false;
+        const waiters = this.dispatchIdleWaiters;
+        this.dispatchIdleWaiters = [];
+        for (const resolve of waiters) {
+          resolve();
+        }
+      });
+  }
+
+  /** Resolve after every queued dispatch has settled. */
+  protected waitForPendingDispatches(): Promise<void> {
+    if (!this.dispatchInFlight) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this.dispatchIdleWaiters.push(resolve);
+    });
   }
 
   /** `protected` so a test can observe what would be sent without sending it. */
@@ -174,6 +281,7 @@ export class LiveActivityReporter {
     phase: LiveActivityPhase,
     event: "update" | "end",
     detail: string,
+    signal: AbortSignal,
   ): Promise<void> {
     try {
       const client = await VellumPlatformClient.create();
@@ -192,6 +300,7 @@ export class LiveActivityReporter {
           event,
           detail,
         }),
+        signal,
       });
       if (!response.ok) {
         log.debug(

package/src/live-voice/live-voice-manager.ts

@@ -11,6 +11,8 @@
 
 import { createRequire } from "node:module";
 
+import { isSessionGroupsEnabled } from "../config/session-groups-gate.js";
+import type { SubagentParentNotification } from "../subagent/parent-notification.js";
 import { getLogger } from "../util/logger.js";
 import { LiveVoiceSessionManager } from "./live-voice-session-manager.js";
 
@@ -24,6 +26,15 @@ type LiveVoiceSessionFactory =
 let manager: LiveVoiceSessionManager | null = null;
 let bundledSessionFactory: LiveVoiceSessionFactory | null = null;
 
+export function deliverSubagentNotificationToLiveVoice(
+  conversationId: string,
+  notification: SubagentParentNotification,
+): boolean {
+  return (
+    manager?.deliverSubagentNotification(conversationId, notification) ?? false
+  );
+}
+
 export function setBundledLiveVoiceSessionFactory(
   factory: LiveVoiceSessionFactory | null,
 ): void {
@@ -50,7 +61,31 @@ export function getLiveVoiceSessionManager(): LiveVoiceSessionManager {
           (
             require("./live-voice-session.js") as typeof import("./live-voice-session.js")
           ).createLiveVoiceSession;
-        return createSession(context);
+        const modeSessionOptions = isSessionGroupsEnabled()
+          ? {
+              acquireModeSessionResidency: async (conversationId: string) => {
+                const { ensureConversationExists } =
+                  require("../persistence/conversation-crud.js") as typeof import("../persistence/conversation-crud.js");
+                const { publishConversationListAndMetadataChanged } =
+                  require("../runtime/sync/resource-sync-events.js") as typeof import("../runtime/sync/resource-sync-events.js");
+                if (ensureConversationExists(conversationId, "vellum")) {
+                  publishConversationListAndMetadataChanged(
+                    "created",
+                    conversationId,
+                  );
+                }
+                const { getOrCreateConversation } =
+                  require("../daemon/conversation-store.js") as typeof import("../daemon/conversation-store.js");
+                const conversation =
+                  await getOrCreateConversation(conversationId);
+                return {
+                  coordinator: conversation.modeSessions,
+                  release: conversation.acquireLiveVoiceResidency(),
+                };
+              },
+            }
+          : {};
+        return createSession(context, modeSessionOptions);
       },
       // The manager reclaims a slot on its own only when a session stopped
       // behaving: either its teardown overran the close budget, or its client