@runtypelabs/persona 4.24.0 → 4.25.0

package/src/types.ts

@@ -4614,6 +4614,24 @@ export type AgentWidgetStatusIndicatorConfig = {
   resumingText?: string;
 };
 
+/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
+export type AgentWidgetVoicePrewarmTrigger = 'hover';
+
+/**
+ * Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
+ * button, and again when the button is rebuilt or `update()` swaps in a
+ * different function. Call `warm()` whenever the visitor shows intent to talk
+ * (it is safe to call repeatedly). Return a cleanup to run on replacement.
+ */
+export type AgentWidgetVoicePrewarmHook = (context: {
+  /** Fire-and-forget voice warm-up; latched and error-swallowing. */
+  warm: () => void;
+  /** The composer's current mic button. */
+  micButton: HTMLButtonElement;
+  /** The widget's mount element. */
+  mount: HTMLElement;
+}) => void | (() => void);
+
 export type AgentWidgetVoiceRecognitionConfig = {
   /**
    * Enable the mic button and voice input. The mic still only renders when the
@@ -4685,6 +4703,30 @@ export type AgentWidgetVoiceRecognitionConfig = {
    */
   onMetrics?: (metrics: VoiceMetrics) => void;
 
+  /**
+   * Warm the voice path when the visitor shows intent to talk, before the mic
+   * click, so the call starts faster.
+   *
+   * - `'hover'`: when the pointer enters the mic button, it receives focus, or
+   *   (on touch) the first pointer-down on it.
+   * - `false`: never.
+   * - A function: custom trigger. Called with `{ warm, micButton, mount }`
+   *   while the mic button exists (and again when the button is rebuilt or
+   *   `update()` swaps in a different function); call `warm()` whenever you
+   *   decide, e.g. when the panel opens. May return a cleanup that runs on
+   *   destroy, rebuild, or replacement. Replaces the built-in trigger.
+   *
+   * What `warm()` does depends on the provider: `runtype` fires its prewarm
+   * (see `provider.runtype.prewarmMode`), a `custom` provider's optional
+   * {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
+   * starts the client-token session init (as `sessionInit` does), since speech
+   * recognition itself cannot start before the click. Warming is
+   * fire-and-forget, latched, and never surfaces an error or UI state.
+   *
+   * @default 'hover' in client token mode, otherwise `false`
+   */
+  prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
+
   // Voice provider configuration
   provider?: {
     type: 'browser' | 'runtype' | 'custom';
@@ -4707,6 +4749,23 @@ export type AgentWidgetVoiceRecognitionConfig = {
        * `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
        */
       createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
+      /**
+       * How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
+       *
+       * - `'request'` (default): a fire-and-forget
+       *   `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
+       * - `'attach'`: open the voice WebSocket ahead of the click without
+       *   starting a call; the click then starts the call on that socket. It can
+       *   create server work before the visitor clicks, so it is opt-in.
+       * @default 'request'
+       */
+      prewarmMode?: 'request' | 'attach';
+      /**
+       * `'attach'` mode: how long (ms) an attached socket waits for the click
+       * before it closes. The server clamps it to 30000..600000.
+       * @default 30000
+       */
+      attachIdleMs?: number;
       /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
       pauseDuration?: number;
       /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
@@ -5014,6 +5073,10 @@ export type VoiceConfig = {
      * `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
      */
     createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
+    /** See `voiceRecognition.provider.runtype.prewarmMode`. @default 'request' */
+    prewarmMode?: 'request' | 'attach';
+    /** See `voiceRecognition.provider.runtype.attachIdleMs`. @default 30000 */
+    attachIdleMs?: number;
     /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
     pauseDuration?: number;
     /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
@@ -5085,6 +5148,14 @@ export interface VoiceProvider {
 
   /** Stop playback / cancel in-flight request without starting recording */
   stopPlayback?(): void;
+
+  /**
+   * Warm the provider ahead of `startListening()` (the widget calls it on mic
+   * intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
+   * throw, emit errors or status, or acquire the microphone. May be called
+   * repeatedly; throttle as needed.
+   */
+  prewarm?(): void;
 }
 
 /**

package/src/ui.ts

@@ -16,6 +16,7 @@ import {
   AgentWidgetConfig,
   AgentWidgetConfigPatch,
   AgentWidgetSessionInitHook,
+  AgentWidgetVoicePrewarmHook,
   AgentWidgetApprovalDecisionOptions,
   AgentWidgetMessage,
   AgentWidgetEvent,
@@ -12981,6 +12982,62 @@ export const createAgentExperience = (
 
   composerVoiceBridge = handleMicButtonClick;
 
+  // Voice prewarm on mic intent (`voiceRecognition.prewarm`). Delegated on the
+  // mount so composer rebuilds stay covered, and `config` is read at event time
+  // so `update()` applies without re-wiring. Default: 'hover' in client token
+  // mode, off otherwise.
+  const warmVoice = () => session.prewarmVoice();
+  const voicePrewarmSetting = () => {
+    const voice = config.voiceRecognition;
+    if (voice?.enabled !== true) return false;
+    return voice.prewarm ?? (config.clientToken ? "hover" : false);
+  };
+  // `pointerover` covers mouse hover and precedes a touch's pointer-down;
+  // `focusin` covers keyboard focus.
+  const handleVoicePrewarmIntent = (event: Event) => {
+    if (voicePrewarmSetting() !== "hover") return;
+    if (micButton?.contains(event.target as Node)) warmVoice();
+  };
+  const voicePrewarmEvents = ["pointerover", "pointerdown", "focusin"];
+  voicePrewarmEvents.forEach((type) => mount.addEventListener(type, handleVoicePrewarmIntent));
+  // A custom prewarm hook is bound to one mic button: it re-runs (after its
+  // cleanup) when the button is rebuilt or `update()` swaps the function.
+  let voicePrewarmHook: AgentWidgetVoicePrewarmHook | undefined;
+  let voicePrewarmHookButton: HTMLButtonElement | null = null;
+  let voicePrewarmHookCleanup: ReturnType<AgentWidgetVoicePrewarmHook>;
+  // Host hook code may throw; neither setup nor cleanup may break the widget.
+  const releaseVoicePrewarmHook = () => {
+    const cleanup = voicePrewarmHookCleanup;
+    voicePrewarmHookCleanup = undefined;
+    try {
+      if (typeof cleanup === "function") cleanup();
+    } catch {
+      // ignored, see above
+    }
+  };
+  const syncVoicePrewarmHook = () => {
+    const setting = voicePrewarmSetting();
+    const next = typeof setting === "function" && micButton ? setting : undefined;
+    const button = next ? micButton : null;
+    if (next === voicePrewarmHook && button === voicePrewarmHookButton) return;
+    releaseVoicePrewarmHook();
+    voicePrewarmHook = next;
+    voicePrewarmHookButton = button;
+    if (!next || !button) return;
+    try {
+      voicePrewarmHookCleanup = next({ warm: warmVoice, micButton: button, mount });
+    } catch {
+      // ignored, see above
+    }
+  };
+  syncVoicePrewarmHook();
+  destroyCallbacks.push(() => {
+    voicePrewarmEvents.forEach((type) =>
+      mount.removeEventListener(type, handleVoicePrewarmIntent)
+    );
+    releaseVoicePrewarmHook();
+  });
+
   if (micButton) {
     // The click listener itself is registered by `wireComposerSurface`, so a
     // composer rebuild unwires it with the rest of the surface.
@@ -13578,6 +13635,7 @@ export const createAgentExperience = (
     if (micButton) {
       composerBindings.addListener(micButton, "click", handleMicButtonClick);
     }
+    syncVoicePrewarmHook();
   };
 
   wireComposerSurface();
@@ -14764,6 +14822,7 @@ export const createAgentExperience = (
           }
         }
       }
+      syncVoicePrewarmHook();
 
       // Update attachment button visibility based on attachments config
       const attachmentsEnabled = config.attachments?.enabled === true;

package/src/ui.voice-prewarm.test.ts

@@ -0,0 +1,222 @@
+// @vitest-environment jsdom
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { createAgentExperience } from "./ui";
+import type {
+  AgentWidgetConfig,
+  AgentWidgetVoicePrewarmHook,
+  AgentWidgetVoiceRecognitionConfig,
+} from "./types";
+
+let controller: ReturnType<typeof createAgentExperience> | undefined;
+
+beforeEach(() => {
+  window.scrollTo = vi.fn();
+});
+
+afterEach(() => {
+  controller?.destroy();
+  controller = undefined;
+  document.body.replaceChildren();
+  vi.restoreAllMocks();
+  vi.unstubAllGlobals();
+});
+
+const initResponse = () =>
+  Response.json({
+    sessionId: "session-1",
+    expiresAt: new Date(Date.now() + 600_000).toISOString(),
+    flow: { id: "agent-1", name: "Agent", description: null },
+    config: { welcomeMessage: null, placeholder: "Ask...", theme: null },
+  });
+
+const runtypeVoice = (
+  extra: Partial<AgentWidgetVoiceRecognitionConfig> = {}
+): AgentWidgetVoiceRecognitionConfig => ({
+  enabled: true,
+  provider: { type: "runtype", runtype: { agentId: "agent-1" } },
+  ...extra,
+});
+
+function mountWidget(config: Partial<AgentWidgetConfig>) {
+  const prewarms: Array<{ url: string; init?: RequestInit }> = [];
+  const inits: string[] = [];
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async (url: string, init?: RequestInit) => {
+      if (url.endsWith("/voice/prewarm")) {
+        prewarms.push({ url, init });
+        return new Response(null, { status: 204 });
+      }
+      if (url.endsWith("/v1/client/init")) {
+        inits.push(url);
+        return initResponse();
+      }
+      return new Response('data: {"type":"done"}\n\n', {
+        headers: { "Content-Type": "text/event-stream" },
+      });
+    })
+  );
+  const mount = document.createElement("div");
+  document.body.appendChild(mount);
+  controller = createAgentExperience(mount, {
+    apiUrl: "https://api.example.com",
+    agentId: "agent-1",
+    launcher: { enabled: false },
+    persistState: false,
+    ...config,
+  } as AgentWidgetConfig);
+  const mic = () => mount.querySelector<HTMLButtonElement>("[data-persona-composer-mic]")!;
+  const hover = () => mic().dispatchEvent(new Event("pointerover", { bubbles: true }));
+  return { mount, mic, hover, prewarms, inits };
+}
+
+// Lets the lazy voice runtime install and any fire-and-forget fetch run.
+const settle = async () => {
+  for (let i = 0; i < 5; i += 1) await new Promise((r) => setTimeout(r, 0));
+};
+
+describe("voiceRecognition.prewarm: default", () => {
+  it("prewarms once on mic hover in client token mode (latched)", async () => {
+    const w = mountWidget({ clientToken: "ct_test", voiceRecognition: runtypeVoice() });
+    await settle();
+    expect(w.prewarms).toHaveLength(0);
+
+    w.hover();
+    await vi.waitFor(() => expect(w.prewarms).toHaveLength(1));
+    expect(w.prewarms[0].url).toBe(
+      "https://api.example.com/v1/client/agents/agent-1/voice/prewarm"
+    );
+    expect(w.prewarms[0].init?.headers).toEqual({ Authorization: "Bearer ct_test" });
+
+    w.hover();
+    w.mic().dispatchEvent(new Event("pointerdown", { bubbles: true }));
+    w.mic().dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
+    await settle();
+    expect(w.prewarms).toHaveLength(1);
+  });
+
+  it("also prewarms on keyboard focus", async () => {
+    const w = mountWidget({ clientToken: "ct_test", voiceRecognition: runtypeVoice() });
+    await settle();
+    w.mic().dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
+    await vi.waitFor(() => expect(w.prewarms).toHaveLength(1));
+  });
+
+  it("ignores hover elsewhere in the widget", async () => {
+    const w = mountWidget({ clientToken: "ct_test", voiceRecognition: runtypeVoice() });
+    await settle();
+    w.mount
+      .querySelector("[data-persona-composer-input]")!
+      .dispatchEvent(new Event("pointerover", { bubbles: true }));
+    await settle();
+    expect(w.prewarms).toHaveLength(0);
+  });
+
+  it("does not auto-prewarm outside client token mode", async () => {
+    const w = mountWidget({
+      voiceRecognition: {
+        enabled: true,
+        provider: { type: "runtype", runtype: { agentId: "agent-1", clientToken: "ct_voice" } },
+      },
+    });
+    await settle();
+    w.hover();
+    await settle();
+    expect(w.prewarms).toHaveLength(0);
+  });
+
+  it("warms the client session on the browser (Web Speech) path", async () => {
+    vi.stubGlobal("SpeechRecognition", vi.fn());
+    const w = mountWidget({ clientToken: "ct_test", voiceRecognition: { enabled: true } });
+    await settle();
+    expect(w.inits).toHaveLength(0);
+    w.hover();
+    await vi.waitFor(() => expect(w.inits).toHaveLength(1));
+    expect(w.prewarms).toHaveLength(0);
+  });
+});
+
+describe("voiceRecognition.prewarm: explicit settings", () => {
+  it("false opts out", async () => {
+    const w = mountWidget({
+      clientToken: "ct_test",
+      voiceRecognition: runtypeVoice({ prewarm: false }),
+    });
+    await settle();
+    w.hover();
+    await settle();
+    expect(w.prewarms).toHaveLength(0);
+  });
+
+  it("'hover' enables it outside client token mode", async () => {
+    const w = mountWidget({
+      voiceRecognition: {
+        enabled: true,
+        prewarm: "hover",
+        provider: { type: "runtype", runtype: { agentId: "agent-1", clientToken: "ct_voice" } },
+      },
+    });
+    await settle();
+    w.hover();
+    await vi.waitFor(() => expect(w.prewarms).toHaveLength(1));
+    expect(w.prewarms[0].init?.headers).toEqual({ Authorization: "Bearer ct_voice" });
+  });
+
+  it("a custom hook receives warm and the mic button, replaces hover, and is cleaned up", async () => {
+    const cleanup = vi.fn();
+    let context: Parameters<AgentWidgetVoicePrewarmHook>[0] | undefined;
+    const hook = vi.fn((ctx: Parameters<AgentWidgetVoicePrewarmHook>[0]) => {
+      context = ctx;
+      return cleanup;
+    });
+    const w = mountWidget({
+      clientToken: "ct_test",
+      voiceRecognition: runtypeVoice({ prewarm: hook }),
+    });
+    await settle();
+    expect(hook).toHaveBeenCalledOnce();
+    expect(context?.micButton).toBe(w.mic());
+    expect(context?.mount).toBe(w.mount);
+
+    w.hover();
+    await settle();
+    expect(w.prewarms).toHaveLength(0);
+
+    context!.warm();
+    await vi.waitFor(() => expect(w.prewarms).toHaveLength(1));
+
+    controller!.destroy();
+    controller = undefined;
+    expect(cleanup).toHaveBeenCalledOnce();
+  });
+
+  it("update() swaps the hook and turning prewarm off releases it", async () => {
+    const cleanupA = vi.fn();
+    const hookA = vi.fn(() => cleanupA);
+    const hookB = vi.fn();
+    mountWidget({ clientToken: "ct_test", voiceRecognition: runtypeVoice({ prewarm: hookA }) });
+    await settle();
+
+    controller!.update({ voiceRecognition: runtypeVoice({ prewarm: hookB }) });
+    expect(cleanupA).toHaveBeenCalledOnce();
+    expect(hookB).toHaveBeenCalledOnce();
+
+    controller!.update({ voiceRecognition: runtypeVoice({ prewarm: false }) });
+    expect(hookB).toHaveBeenCalledOnce();
+    expect(hookA).toHaveBeenCalledOnce();
+  });
+
+  it("a throwing hook does not break mount", async () => {
+    const w = mountWidget({
+      clientToken: "ct_test",
+      voiceRecognition: runtypeVoice({
+        prewarm: () => {
+          throw new Error("host bug");
+        },
+      }),
+    });
+    await settle();
+    expect(w.mic()).toBeTruthy();
+  });
+});