@runtypelabs/persona 4.25.0 → 4.26.1

package/src/voice/fallback-speech-engine.ts

@@ -30,27 +30,33 @@ export class FallbackSpeechEngine implements SpeechEngine {
 
   // Whichever engine is currently driving playback, so pause/resume/stop route
   // to the right one after a fallback has (or hasn't) kicked in.
-  private active: SpeechEngine;
+  #active: SpeechEngine;
 
+  readonly #primary: SpeechEngine;
+  readonly #fallback: SpeechEngine;
+  readonly #options: FallbackSpeechEngineOptions;
   constructor(
-    private readonly primary: SpeechEngine,
-    private readonly fallback: SpeechEngine,
-    private readonly options: FallbackSpeechEngineOptions = {},
+    primary: SpeechEngine,
+    fallback: SpeechEngine,
+    options: FallbackSpeechEngineOptions = {},
   ) {
-    this.active = primary;
+    this.#primary = primary;
+    this.#fallback = fallback;
+    this.#options = options;
+    this.#active = primary;
   }
 
   // Pause/resume only matters once something is playing, and both built-in
   // engines support it; report the active engine's capability.
   get supportsPause(): boolean {
-    return this.active.supportsPause;
+    return this.#active.supportsPause;
   }
 
   speak(request: SpeechRequest, callbacks: SpeechCallbacks): void {
-    this.active = this.primary;
+    this.#active = this.#primary;
     let started = false;
 
-    this.primary.speak(request, {
+    this.#primary.speak(request, {
       onStart: () => {
         started = true;
         callbacks.onStart?.();
@@ -63,27 +69,27 @@ export class FallbackSpeechEngine implements SpeechEngine {
           return;
         }
         // Pre-start failure: silently hand the utterance to the fallback.
-        this.options.onFallback?.(error);
-        this.active = this.fallback;
-        this.fallback.speak(request, callbacks);
+        this.#options.onFallback?.(error);
+        this.#active = this.#fallback;
+        this.#fallback.speak(request, callbacks);
       },
     });
   }
 
   pause(): void {
-    this.active.pause();
+    this.#active.pause();
   }
 
   resume(): void {
-    this.active.resume();
+    this.#active.resume();
   }
 
   stop(): void {
-    this.active.stop();
+    this.#active.stop();
   }
 
   destroy(): void {
-    this.primary.destroy?.();
-    this.fallback.destroy?.();
+    this.#primary.destroy?.();
+    this.#fallback.destroy?.();
   }
 }

package/src/voice/keyed-voice-transcript.ts

@@ -0,0 +1,314 @@
+import type { AgentWidgetMessage, InjectMessageOptions } from "../types";
+
+/** The session surface the keyed reconciler drives. */
+export interface KeyedVoiceTranscriptHost {
+  find(id: string): AgentWidgetMessage | undefined;
+  inject(options: InjectMessageOptions): AgentWidgetMessage;
+  upsert(message: AgentWidgetMessage): void;
+  /** Clear `streaming` / `voiceProcessing` on these messages in one update. */
+  settle(ids: Set<string>): void;
+  setStreaming(streaming: boolean): void;
+  /** The provider already played this reply's audio: browser TTS must skip it. */
+  markSpoken(id: string): void;
+}
+
+/** Bubbles owned by one provider turnId. */
+type KeyedTurn = {
+  userId?: string;
+  assistantId?: string;
+  userFinal?: boolean;
+  assistantFinal?: boolean;
+  userStartMs?: number;
+  assistantStartMs?: number;
+  /** Submitted as a chat turn: the chat pipeline renders its reply. */
+  claimed?: boolean;
+};
+
+/** Loose text match between a transcript and the delegated request text. */
+const normalize = (text: string) => text.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
+
+/**
+ * Reconciles turn-keyed voice transcripts (full-duplex providers such as
+ * GPT-Live) into chat bubbles: each `(turnId, role)` owns one bubble that later
+ * frames replace in place, so overlapping turns stay separate and may arrive in
+ * any order. No empty assistant placeholder is injected; the session's
+ * `streaming` flag drives the standalone typing indicator instead. A new bubble
+ * with a `startMs` sorts above any keyed bubble of the call that started later,
+ * so a late user transcript still renders above the reply it prompted.
+ *
+ * Cancelled output is the provider's to drop (until the server acknowledges
+ * the cancel): every reply that reaches `apply` after a stop renders, which
+ * keeps a later answer (e.g. a delegation result) visible while its audio plays.
+ *
+ * Ships in the lazy voice-runtime chunk: only sessions with a voice provider
+ * ever construct one.
+ */
+export class KeyedVoiceTranscript {
+  private turns = new Map<string, KeyedTurn>();
+  private latestId: string | null = null;
+  private cancelled = new Set<string>();
+
+  constructor(private host: KeyedVoiceTranscriptHost) {}
+
+  apply(
+    role: "user" | "assistant",
+    text: string,
+    isFinal: boolean,
+    turnId: string,
+    startMs?: number,
+    caption?: boolean,
+  ): void {
+    const host = this.host;
+    // Speech-to-speech transcripts can carry a leading space (" What are…").
+    text = text.trim();
+    let turn = this.turns.get(turnId);
+    if (!turn) {
+      turn = {};
+      this.turns.set(turnId, turn);
+      this.latestId = turnId;
+    }
+
+    if (role === "user") {
+      turn.userFinal = isFinal;
+      const existing = turn.userId ? host.find(turn.userId) : undefined;
+      if (existing) {
+        const moved = turn.userStartMs === undefined && startMs !== undefined;
+        if (moved) turn.userStartMs = startMs;
+        host.upsert({
+          ...existing,
+          content: text,
+          voiceProcessing: !isFinal,
+          ...(moved && this.placeBefore(this.startedAfter(startMs))),
+        });
+      } else {
+        // A user transcript that lands after its own turn's reply started still
+        // renders above that reply: borrow the reply's timestamp, sort just ahead.
+        turn.userStartMs = startMs;
+        const reply = turn.assistantId ? host.find(turn.assistantId) : undefined;
+        turn.userId = host.inject({
+          role: "user",
+          content: text,
+          streaming: false,
+          voiceProcessing: !isFinal,
+          voiceCaption: caption,
+          ...this.placeBefore(this.startedAfter(startMs) ?? reply),
+        }).id;
+      }
+    } else if (!this.cancelled.has(turnId)) {
+      turn.assistantFinal = isFinal;
+      const existing = turn.assistantId ? host.find(turn.assistantId) : undefined;
+      if (existing) {
+        // A timestamp that arrives after the bubble did re-positions it.
+        const moved = turn.assistantStartMs === undefined && startMs !== undefined;
+        if (moved) turn.assistantStartMs = startMs;
+        host.upsert({
+          ...existing,
+          content: text,
+          streaming: !isFinal,
+          voiceProcessing: !isFinal,
+          ...(moved && this.placeBefore(this.startedAfter(startMs))),
+        });
+      } else if (text) {
+        turn.assistantStartMs = startMs;
+        turn.assistantId = host.inject({
+          role: "assistant",
+          content: text,
+          streaming: !isFinal,
+          voiceProcessing: !isFinal,
+          voiceCaption: caption,
+          ...this.placeBefore(this.startedAfter(startMs)),
+        }).id;
+      }
+      if (isFinal && turn.assistantId) host.markSpoken(turn.assistantId);
+    }
+    this.sync();
+  }
+
+  /**
+   * The user bubble a delegated request came from, claimed so it is submitted
+   * once. With utterance ids it is the last one's bubble; one that hasn't been
+   * transcribed yet is created now (from `userText`), so its transcript later
+   * fills it instead of adding a second bubble. Without an id it is the newest
+   * unclaimed bubble whose text matches exactly, else one where either text
+   * is a prefix of the other (the request can beat the final transcript).
+   * `null` when nothing matches: never someone else's bubble.
+   *
+   * `userUtteranceIds` lists every utterance a split request joins (the last
+   * is the one submitted). The others are claimed too: they stay displayed as
+   * captions, never sent and never claimed again. One not transcribed yet
+   * is reserved, so its transcript later renders as a caption.
+   */
+  claimUserTurn(userText: string, userUtteranceIds: string[] = []): string | null {
+    const requestId = userUtteranceIds[userUtteranceIds.length - 1];
+    for (const id of userUtteranceIds) {
+      if (id === requestId) continue;
+      if (!this.turns.has(id)) this.turns.set(id, {});
+      this.turns.get(id)!.claimed = true;
+    }
+    let pick: KeyedTurn | undefined;
+    if (requestId) {
+      pick = this.turns.get(requestId);
+      if (!pick) {
+        pick = {};
+        this.turns.set(requestId, pick);
+      }
+      if (pick.claimed) return null;
+      if (!pick.userId || !this.host.find(pick.userId)) {
+        pick.userFinal = true;
+        pick.userId = this.host.inject({ role: "user", content: userText.trim() }).id;
+      }
+    } else {
+      const want = normalize(userText);
+      const open = [...this.turns.values()]
+        .reverse()
+        .flatMap((turn) => {
+          const bubble = turn.userId && !turn.claimed ? this.host.find(turn.userId) : undefined;
+          return bubble ? [{ turn, text: normalize(bubble.content) }] : [];
+        })
+        .filter(({ text }) => text && want);
+      pick = (
+        open.find(({ text }) => text === want) ??
+        open.find(({ text }) => text.startsWith(want) || want.startsWith(text))
+      )?.turn;
+    }
+    if (!pick) return null;
+    // No sync: the chat turn about to start owns the streaming flag.
+    pick.claimed = true;
+    return pick.userId!;
+  }
+
+  /** Explicit stop: drop the rest of every in-flight reply, else the awaited one. */
+  cancel(): void {
+    let targets = this.inFlight();
+    if (targets.length === 0) {
+      const awaiting = this.awaiting();
+      if (!awaiting) return;
+      targets = [awaiting];
+    }
+    for (const id of targets) this.cancelled.add(id);
+    this.closeReplies(targets);
+    this.sync();
+  }
+
+  /**
+   * Voice error: close every in-flight reply as-is and show the error text (as
+   * the awaited turn's reply, when one is waiting), so `streaming` clears.
+   */
+  fail(errorText: string): void {
+    const awaitingId = this.awaiting();
+    const inFlight = this.inFlight();
+    if (!awaitingId && inFlight.length === 0) return;
+    for (const id of inFlight) this.turns.get(id)!.assistantFinal = true;
+    this.closeReplies(inFlight);
+    const msg = this.host.inject({
+      role: "assistant",
+      content: errorText,
+      streaming: false,
+      voiceProcessing: false,
+    });
+    const awaiting = awaitingId ? this.turns.get(awaitingId) : undefined;
+    if (awaiting) {
+      awaiting.assistantId = msg.id;
+      awaiting.assistantFinal = true;
+    }
+    this.sync();
+  }
+
+  /** Call ended: freeze every keyed bubble as-is and forget the turns. */
+  settle(): void {
+    if (this.turns.size === 0) return;
+    const wasPending = this.pending();
+    const ids = new Set<string>();
+    for (const turn of this.turns.values()) {
+      if (turn.userId) ids.add(turn.userId);
+      if (turn.assistantId) {
+        ids.add(turn.assistantId);
+        this.host.markSpoken(turn.assistantId);
+      }
+    }
+    this.turns.clear();
+    this.latestId = null;
+    this.host.settle(ids);
+    if (wasPending) this.host.setStreaming(false);
+  }
+
+  /** The call's earliest-starting keyed bubble that began after `startMs`. */
+  private startedAfter(startMs: number | undefined): AgentWidgetMessage | undefined {
+    if (startMs === undefined) return undefined;
+    let best: { start: number; id: string } | undefined;
+    for (const turn of this.turns.values()) {
+      for (const [id, start] of [
+        [turn.userId, turn.userStartMs],
+        [turn.assistantId, turn.assistantStartMs],
+      ] as const) {
+        if (id && start !== undefined && start > startMs && (!best || start < best.start)) {
+          best = { start, id };
+        }
+      }
+    }
+    return best && this.host.find(best.id);
+  }
+
+  /**
+   * Sort a bubble just ahead of `next` (messages order by time, then
+   * sequence): halfway between `next` and the keyed bubbles already placed
+   * ahead of it. Those all started earlier (else one of them would be
+   * `next`), so bubbles placed ahead of the same reply stay in `startMs`
+   * order. Sequences are timestamp-sized, so halving, not a tiny fixed
+   * offset, keeps them distinct.
+   */
+  private placeBefore(next: AgentWidgetMessage | undefined) {
+    if (!next) return undefined;
+    const top = next.sequence ?? 0;
+    let low = top - 1;
+    for (const turn of this.turns.values()) {
+      for (const id of [turn.userId, turn.assistantId]) {
+        const placed = id ? this.host.find(id) : undefined;
+        const seq = placed?.sequence;
+        if (placed?.createdAt === next.createdAt && seq !== undefined && seq > low && seq < top) {
+          low = seq;
+        }
+      }
+    }
+    return { createdAt: next.createdAt, sequence: (low + top) / 2 };
+  }
+
+  private closeReplies(turnIds: string[]): void {
+    const ids = new Set<string>();
+    for (const id of turnIds) {
+      const assistantId = this.turns.get(id)?.assistantId;
+      if (assistantId) {
+        ids.add(assistantId);
+        this.host.markSpoken(assistantId);
+      }
+    }
+    if (ids.size > 0) this.host.settle(ids);
+  }
+
+  /** Turns whose assistant bubble is visible but not yet final. */
+  private inFlight(): string[] {
+    const ids: string[] = [];
+    for (const [id, turn] of this.turns) {
+      if (turn.assistantId && !turn.assistantFinal && !this.cancelled.has(id)) ids.push(id);
+    }
+    return ids;
+  }
+
+  /** The newest turn has a final user utterance and no reply text yet. */
+  private awaiting(): string | null {
+    const id = this.latestId;
+    if (!id) return null;
+    const turn = this.turns.get(id);
+    return turn?.userFinal && !turn.assistantId && !turn.claimed && !this.cancelled.has(id)
+      ? id
+      : null;
+  }
+
+  private pending(): boolean {
+    return this.awaiting() !== null || this.inFlight().length > 0;
+  }
+
+  private sync(): void {
+    this.host.setStreaming(this.pending());
+  }
+}

package/src/voice/mic-state-styles.ts

@@ -0,0 +1,182 @@
+/**
+ * Mic-button state styling for session-owned voice providers (Runtype /
+ * custom): recording, processing, speaking, and the reset back to idle.
+ *
+ * Ships in the lazy voice-runtime chunk (exported from `voice-runtime.ts`):
+ * these states only arise from a provider, which `setupVoice` constructs from
+ * that same chunk, so pages without session voice never download it. The UI
+ * passes its live closure state through `MicStateContext`; the icon renderer
+ * comes from core so this chunk never carries its own icon registry.
+ */
+import type { AgentWidgetConfig } from "../types";
+import type { AgentWidgetSession } from "../session";
+
+/** Snapshot of the idle mic, shared with the UI's browser-dictation path. */
+export type OriginalMicStyles = {
+  backgroundColor: string;
+  color: string;
+  borderColor: string;
+  iconName: string;
+  iconSize: number;
+};
+
+export type MicStateContext = {
+  mic: () => HTMLButtonElement | null;
+  config: () => AgentWidgetConfig;
+  session: () => AgentWidgetSession;
+  original: OriginalMicStyles | null;
+  setMicState: (state: "idle" | "recording" | "processing" | "speaking") => void;
+  renderIcon: (name: string, size: number, color: string, strokeWidth: number) => SVGElement | null;
+  iconFallbackPx: number;
+};
+
+export type MicStateStyles = {
+  recording: () => void;
+  processing: () => void;
+  speaking: () => void;
+  reset: () => void;
+};
+
+export const createMicStateStyles = (ctx: MicStateContext): MicStateStyles => {
+  const storeOriginalMicStyles = (micButton: HTMLButtonElement) => {
+    if (ctx.original) return; // Already stored
+    const voiceConfig = ctx.config().voiceRecognition ?? {};
+    ctx.original = {
+      backgroundColor: micButton.style.backgroundColor,
+      color: micButton.style.color,
+      borderColor: micButton.style.borderColor,
+      iconName: voiceConfig.iconName ?? "mic",
+      iconSize: parseFloat(voiceConfig.iconSize ?? "") || ctx.iconFallbackPx,
+    };
+  };
+
+  /** Swap the mic button's SVG icon */
+  const swapMicIcon = (micButton: HTMLButtonElement, iconName: string, color: string) => {
+    const existingSvg = micButton.querySelector("svg");
+    if (existingSvg) existingSvg.remove();
+    const size =
+      ctx.original?.iconSize ??
+      (parseFloat(ctx.config().voiceRecognition?.iconSize ?? "") || ctx.iconFallbackPx);
+    const newSvg = ctx.renderIcon(iconName, size, color, 1.5);
+    if (newSvg) micButton.appendChild(newSvg);
+  };
+
+  /** Remove all voice state CSS classes */
+  const removeAllVoiceStateClasses = (micButton: HTMLButtonElement) => {
+    micButton.classList.remove("persona-voice-recording", "persona-voice-processing", "persona-voice-speaking");
+  };
+
+  const recording = () => {
+    const micButton = ctx.mic();
+    if (!micButton) return;
+    storeOriginalMicStyles(micButton);
+    const voiceConfig = ctx.config().voiceRecognition ?? {};
+    const recordingBackgroundColor = voiceConfig.recordingBackgroundColor;
+    const recordingIconColor = voiceConfig.recordingIconColor;
+    const recordingBorderColor = voiceConfig.recordingBorderColor;
+    removeAllVoiceStateClasses(micButton);
+    micButton.classList.add("persona-voice-recording");
+    ctx.setMicState("recording");
+    micButton.style.backgroundColor = recordingBackgroundColor ?? "var(--persona-voice-recording-bg, #ef4444)";
+    micButton.style.color = recordingIconColor ?? "var(--persona-voice-recording-indicator, #ffffff)";
+    if (recordingIconColor) {
+      const svg = micButton.querySelector("svg");
+      if (svg) svg.setAttribute("stroke", recordingIconColor);
+    }
+    if (recordingBorderColor) micButton.style.borderColor = recordingBorderColor;
+    micButton.setAttribute("aria-label", "Stop voice recognition");
+  };
+
+  const processing = () => {
+    const micButton = ctx.mic();
+    if (!micButton) return;
+    storeOriginalMicStyles(micButton);
+    const voiceConfig = ctx.config().voiceRecognition ?? {};
+    const interruptionMode = ctx.session().getVoiceInterruptionMode();
+    const originalMicStyles = ctx.original;
+    const iconName = voiceConfig.processingIconName ?? "loader";
+    const iconColor = voiceConfig.processingIconColor ?? originalMicStyles?.color ?? "";
+    const bgColor = voiceConfig.processingBackgroundColor ?? originalMicStyles?.backgroundColor ?? "";
+    const borderColor = voiceConfig.processingBorderColor ?? originalMicStyles?.borderColor ?? "";
+
+    removeAllVoiceStateClasses(micButton);
+    micButton.classList.add("persona-voice-processing");
+    ctx.setMicState("processing");
+    micButton.style.backgroundColor = bgColor;
+    micButton.style.borderColor = borderColor;
+    const resolvedColor = iconColor || "currentColor";
+    micButton.style.color = resolvedColor;
+    swapMicIcon(micButton, iconName, resolvedColor);
+    micButton.setAttribute("aria-label", "Processing voice input");
+    // In "none" mode the button is not actionable during processing
+    if (interruptionMode === "none") {
+      micButton.style.cursor = "default";
+    }
+  };
+
+  const speaking = () => {
+    const micButton = ctx.mic();
+    if (!micButton) return;
+    storeOriginalMicStyles(micButton);
+    const voiceConfig = ctx.config().voiceRecognition ?? {};
+    const interruptionMode = ctx.session().getVoiceInterruptionMode();
+    const originalMicStyles = ctx.original;
+    // Default icon depends on interruption mode:
+    // "square" for cancel, "mic" for barge-in (hot mic), "volume-2" otherwise
+    const defaultSpeakingIcon = interruptionMode === "cancel" ? "square"
+      : interruptionMode === "barge-in" ? "mic"
+      : "volume-2";
+    const iconName = voiceConfig.speakingIconName ?? defaultSpeakingIcon;
+    const iconColor = voiceConfig.speakingIconColor
+      ?? (interruptionMode === "barge-in" ? (voiceConfig.recordingIconColor ?? originalMicStyles?.color ?? "") : (originalMicStyles?.color ?? ""));
+    const bgColor = voiceConfig.speakingBackgroundColor
+      ?? (interruptionMode === "barge-in" ? (voiceConfig.recordingBackgroundColor ?? "var(--persona-voice-recording-bg, #ef4444)") : (originalMicStyles?.backgroundColor ?? ""));
+    const borderColor = voiceConfig.speakingBorderColor
+      ?? (interruptionMode === "barge-in" ? (voiceConfig.recordingBorderColor ?? "") : (originalMicStyles?.borderColor ?? ""));
+
+    removeAllVoiceStateClasses(micButton);
+    micButton.classList.add("persona-voice-speaking");
+    ctx.setMicState("speaking");
+    micButton.style.backgroundColor = bgColor;
+    micButton.style.borderColor = borderColor;
+    const resolvedColor = iconColor || "currentColor";
+    micButton.style.color = resolvedColor;
+    swapMicIcon(micButton, iconName, resolvedColor);
+
+    // aria-label varies by interruption mode
+    const ariaLabel = interruptionMode === "cancel"
+      ? "Stop playback and re-record"
+      : interruptionMode === "barge-in"
+      ? "Speak to interrupt"
+      : "Agent is speaking";
+    micButton.setAttribute("aria-label", ariaLabel);
+    // In "none" mode the button is not actionable during speaking
+    if (interruptionMode === "none") {
+      micButton.style.cursor = "default";
+    }
+    // In "barge-in" mode, add recording class to show mic is hot
+    if (interruptionMode === "barge-in") {
+      micButton.classList.add("persona-voice-recording");
+    }
+  };
+
+  /** Restore mic button to idle state (icon, colors, aria-label, cursor) */
+  const reset = () => {
+    const micButton = ctx.mic();
+    if (!micButton) return;
+    removeAllVoiceStateClasses(micButton);
+    ctx.setMicState("idle");
+    const originalMicStyles = ctx.original;
+    if (originalMicStyles) {
+      micButton.style.backgroundColor = originalMicStyles.backgroundColor ?? "";
+      micButton.style.color = originalMicStyles.color ?? "";
+      micButton.style.borderColor = originalMicStyles.borderColor ?? "";
+      swapMicIcon(micButton, originalMicStyles.iconName, originalMicStyles.color || "currentColor");
+      ctx.original = null;
+    }
+    micButton.style.cursor = "";
+    micButton.setAttribute("aria-label", "Start voice recognition");
+  };
+
+  return { recording, processing, speaking, reset };
+};