@iloveagents/foundry-web-voice 0.1.12 → 0.1.14

package/README.md

@@ -43,6 +43,23 @@ retrieval surface, and the reply is spoken back in the Azure voice you configure
 that is `turn_detection.create_response: false` plus `pre_generated_assistant_message` — both
 documented Voice Live features, not tricks.
 
+For slow app-agent runs, opt into `relayInterimResponse: { text: "One moment, please.",
+delayMs: 15000, skipInitialResponse: true }`. The relay speaks it once while waiting
+for answer text, through the same serialized speech queue. `skipInitialResponse`
+lets the first reply of each call open the conversation without an acknowledgement.
+Answer text (including a buffered short reply), interruption, run completion and
+disconnect cancel a pending acknowledgement. It is presentation feedback, so it
+does not add a message to the persisted conversation. Native Voice Live interim
+responses cannot observe tools executing in the app's separate agent.
+
+Optionally provide `describeActivity({ toolName, completed, args })` to return a
+short description of observed tool work, or `null` for unknown activity. The
+watcher reports only tools from the current user turn, preferring pending calls.
+With `intervalMs` (default 18000), changed descriptions can be spoken occasionally,
+never repeating a phrase and capped at three updates per run. Described tool work
+can receive an update during the opening turn; the generic fallback remains
+suppressed there. Curate descriptions rather than speaking raw arguments or results.
+
 **`realtime`** hands the conversation to the Voice Live model for the lowest possible latency. It
 can still call your client-side tools: the schemas in `clientToolRegistry` are sent with the
 session, and a call is dispatched through the same registry the typed chat uses, so `ui_navigate`

package/dist/adapter/relay-activity.d.ts

@@ -0,0 +1,4 @@
+import type { ThreadMessage } from "@assistant-ui/react";
+import type { RelayActivity } from "./types.js";
+/** Only describe work in the current user turn; prefer a call still in progress. */
+export declare function currentRelayActivity(messages: readonly ThreadMessage[]): RelayActivity | null;

package/dist/adapter/relay-activity.js

@@ -0,0 +1,17 @@
+/** Only describe work in the current user turn; prefer a call still in progress. */
+export function currentRelayActivity(messages) {
+    let userIndex = messages.length - 1;
+    while (userIndex >= 0 && messages[userIndex]?.role !== "user")
+        userIndex--;
+    if (userIndex < 0)
+        return null;
+    const tools = messages
+        .slice(userIndex + 1)
+        .flatMap((message) => message.role === "assistant"
+        ? message.content.filter((part) => part.type === "tool-call")
+        : []);
+    const tool = [...tools].reverse().find((part) => part.result === undefined) ?? tools[tools.length - 1];
+    return tool
+        ? { toolName: tool.toolName, completed: tool.result !== undefined, args: tool.args }
+        : null;
+}

package/dist/adapter/types.d.ts

@@ -31,9 +31,31 @@ export interface VoiceTranscript {
     text: string;
     isFinal: boolean;
 }
+export interface RelayActivity {
+    toolName: string;
+    completed: boolean;
+    args?: Readonly<Record<string, unknown>>;
+}
 export interface VoiceConfig {
     /** @default "relay" */
     mode?: VoiceMode;
+    /**
+     * Spoken progress when the app's agent takes longer to answer.
+     * Relay only: Voice Live cannot observe tools running in the app's agent.
+     * Uses the normal speech queue, never adds a message to conversation history.
+     * Cancelled on the first answer text, interruption, run end or disconnect.
+     */
+    relayInterimResponse?: {
+        text: string;
+        /** @default 15000 */
+        delayMs?: number;
+        /** Suppress opening-turn filler; described tool work can still receive updates. */
+        skipInitialResponse?: boolean;
+        /** Describe observed tool activity without speaking raw arguments or reasoning. */
+        describeActivity?: (activity: RelayActivity) => string | null;
+        /** Minimum spacing for changed activity updates; at most three updates per run. */
+        intervalMs?: number;
+    };
     /**
      * How to reach Voice Live. In production this is `{ proxyUrl }`: a
      * browser cannot set an `Authorization` header on a WebSocket, so the

package/dist/adapter/voice-bridge.d.ts

@@ -25,7 +25,7 @@
  */
 import { type RealtimeVoiceAdapter } from "@assistant-ui/react";
 import type { VoiceLiveClientEvent, VoiceLiveServerEvent, SessionState } from "@iloveagents/foundry-voice-live-react";
-import type { VoiceConfig } from "./types.js";
+import type { RelayActivity, VoiceConfig } from "./types.js";
 /** What the bridge needs from the live session; supplied by the React host. */
 export interface VoiceTransportControls {
     connect: () => Promise<void>;
@@ -74,6 +74,13 @@ export declare class VoiceBridge {
      * stream, and speak the whole reply again. Forever.
      */
     private answerDone;
+    private hasSpokenAnswer;
+    private relayRunId;
+    private interimTimer;
+    private interimActivity;
+    private interimSuppressed;
+    private interimCount;
+    private interimTexts;
     /** The response currently being spoken, once the service has named it. */
     private activeResponseId;
     /**
@@ -194,6 +201,11 @@ export declare class VoiceBridge {
      * the moment the session goes live.
      */
     primeAnswer(messageId: string | null): void;
+    /** A stable user-turn id while the agent runs; null on completion/cancellation. */
+    trackRelayRun(runId: string | null): void;
+    trackRelayActivity(activity: RelayActivity | null): void;
+    private scheduleInterim;
+    private clearInterim;
     trackAnswer(messageId: string, text: string, isComplete: boolean): void;
     /**
      * Speak exact text through Voice Live without invoking its model.

package/dist/adapter/voice-bridge.js

@@ -62,6 +62,13 @@ export class VoiceBridge {
          * stream, and speak the whole reply again. Forever.
          */
         this.answerDone = false;
+        this.hasSpokenAnswer = false;
+        this.relayRunId = null;
+        this.interimTimer = null;
+        this.interimActivity = null;
+        this.interimSuppressed = false;
+        this.interimCount = 0;
+        this.interimTexts = new Set();
         /** The response currently being spoken, once the service has named it. */
         this.activeResponseId = null;
         /**
@@ -266,6 +273,8 @@ export class VoiceBridge {
     handleServerEvent(event) {
         switch (event.type) {
             case "input_audio_buffer.speech_started":
+                this.interimSuppressed = true;
+                this.clearInterim();
                 // Barge-in. Drop everything still queued to be spoken; the wire's
                 // in-flight response is cancelled by the service (`interrupt_response`),
                 // and cancelling locally flushes playback that is already buffered.
@@ -369,6 +378,8 @@ export class VoiceBridge {
      * is exactly the pair of routes on which the transport dies mid-utterance.
      */
     endCall() {
+        this.clearInterim();
+        this.relayRunId = null;
         this.connected = false;
         // Cleared here rather than on each exit path: the cancellation branch left
         // it set, and a later `handleClosed()` would then act on the controls of a
@@ -384,6 +395,7 @@ export class VoiceBridge {
         this.sentences.reset();
         this.answerId = null;
         this.answerDone = false;
+        this.hasSpokenAnswer = false;
     }
     // ---- relay: speak the agent's answer --------------------------------------
     /**
@@ -402,12 +414,83 @@ export class VoiceBridge {
     primeAnswer(messageId) {
         this.skipAnswerId = messageId;
     }
+    /** A stable user-turn id while the agent runs; null on completion/cancellation. */
+    trackRelayRun(runId) {
+        if (runId === this.relayRunId)
+            return;
+        this.clearInterim();
+        this.relayRunId = runId;
+        this.interimActivity = null;
+        this.interimSuppressed = false;
+        this.interimCount = 0;
+        this.interimTexts.clear();
+        const interim = this.config.relayInterimResponse;
+        if (!runId || !interim?.text.trim() || this.mode !== "relay" || !this.isLive)
+            return;
+        if (interim.skipInitialResponse && !this.hasSpokenAnswer)
+            return;
+        this.scheduleInterim(interim.delayMs ?? 15000);
+    }
+    trackRelayActivity(activity) {
+        this.interimActivity = activity;
+        const config = this.config.relayInterimResponse;
+        if (activity &&
+            config?.describeActivity &&
+            this.interimTimer === null &&
+            this.interimCount === 0 &&
+            this.relayRunId &&
+            !this.interimSuppressed &&
+            this.isLive &&
+            this.mode === "relay") {
+            // Real tool work can merit an update even in the opening turn; greetings cannot.
+            if (config.describeActivity(activity)?.trim())
+                this.scheduleInterim(config.delayMs ?? 15000);
+        }
+    }
+    scheduleInterim(delay) {
+        if (!Number.isFinite(delay) || delay < 0)
+            return;
+        this.interimTimer = setTimeout(() => {
+            this.interimTimer = null;
+            const config = this.config.relayInterimResponse;
+            if (!config || !this.isLive || !this.relayRunId || this.interimSuppressed)
+                return;
+            const activityText = this.interimActivity
+                ? config.describeActivity?.(this.interimActivity)
+                : null;
+            const text = activityText?.trim() ||
+                (this.interimCount === 0 && !(config.skipInitialResponse && !this.hasSpokenAnswer)
+                    ? config.text.trim()
+                    : "");
+            if (text &&
+                !this.interimTexts.has(text) &&
+                !this.speech.isSpeaking &&
+                this.speech.pendingCount === 0) {
+                this.interimTexts.add(text);
+                this.interimCount++;
+                this.speech.enqueue(text);
+            }
+            if (config.describeActivity && this.interimCount < 3)
+                this.scheduleInterim(Math.max(10000, config.intervalMs ?? 18000));
+        }, delay);
+    }
+    clearInterim() {
+        if (this.interimTimer !== null)
+            clearTimeout(this.interimTimer);
+        this.interimTimer = null;
+    }
     trackAnswer(messageId, text, isComplete) {
         // The watcher is mounted for the whole app lifetime, but an answer is
         // only spoken while a voice session is actually live — otherwise every
         // typed conversation would queue speech at a disconnected transport.
         if (this.mode !== "relay" || !this.isLive)
             return;
+        // Short replies are buffered until their sentence/turn completes. Once
+        // answer text is arriving, a waiting message must not jump ahead of it.
+        if (text.trim()) {
+            this.interimSuppressed = true;
+            this.clearInterim();
+        }
         if (messageId === this.skipAnswerId)
             return;
         if (this.answerId !== messageId) {
@@ -423,13 +506,15 @@ export class VoiceBridge {
         // bracket, so it must not be held back — holding it at that point drops
         // it, since `flush()` sees only what was pushed.
         const speakable = toSpeakableText(text, { final: isComplete });
-        for (const utterance of this.sentences.push(speakable))
-            this.speech.enqueue(utterance);
+        const utterances = this.sentences.push(speakable);
         if (isComplete) {
-            for (const utterance of this.sentences.flush())
-                this.speech.enqueue(utterance);
+            utterances.push(...this.sentences.flush());
             this.answerDone = true;
         }
+        for (const utterance of utterances) {
+            this.hasSpokenAnswer = true;
+            this.speech.enqueue(utterance);
+        }
     }
     /**
      * Speak exact text through Voice Live without invoking its model.
@@ -441,6 +526,7 @@ export class VoiceBridge {
      * response gate as every other turn.
      */
     sendSpokenText(text) {
+        this.clearInterim();
         this.controls?.sendEvent({
             type: "response.create",
             response: {

package/dist/index.d.ts

@@ -13,7 +13,7 @@
  */
 export { createVoiceModule, type VoiceChatModule } from "./react/voice-module.js";
 export { FoundryVoice, installVoice, type VoiceInstallOptions, type VoiceInstallation, } from "./react/install.js";
-export type { VoiceConfig, VoiceMode, VoiceTranscript } from "./adapter/types.js";
+export type { RelayActivity, VoiceConfig, VoiceMode, VoiceTranscript } from "./adapter/types.js";
 export { VoiceMicButton } from "./react/voice-mic-button.js";
 export { VoiceStatusStrip } from "./react/voice-status-strip.js";
 export { VoiceStage, type VoiceStageProps } from "./react/voice-stage.js";

package/dist/react/relay-answer-watcher.js

@@ -12,12 +12,15 @@
  */
 import { useEffect, useRef } from "react";
 import { useAuiState, useVoiceState } from "@assistant-ui/react";
+import { currentRelayActivity } from "../adapter/relay-activity.js";
 export function RelayAnswerWatcher({ bridge }) {
     // The array identity changes on every streamed token, which is exactly
     // the update this component exists to observe. It renders nothing, so the
     // re-render costs a comparison and a `useEffect`.
     const messages = useAuiState((s) => s.thread.messages);
     const status = useVoiceState()?.status.type;
+    const running = useAuiState((s) => s.thread.isRunning);
+    const userTurnId = [...messages].reverse().find((m) => m.role === "user")?.id ?? null;
     // Whatever is on screen when the session goes live was answered before the
     // user asked for voice. Reading it back at them is not a greeting, it is a
     // non-sequitur.
@@ -29,6 +32,13 @@ export function RelayAnswerWatcher({ bridge }) {
         const lastAssistant = [...latest.current].reverse().find((m) => m.role === "assistant");
         bridge.primeAnswer(lastAssistant?.id ?? null);
     }, [bridge, status]);
+    useEffect(() => {
+        bridge.trackRelayRun(status === "running" && running ? userTurnId : null);
+        return () => bridge.trackRelayRun(null);
+    }, [bridge, status, running, userTurnId]);
+    useEffect(() => {
+        bridge.trackRelayActivity(status === "running" && running ? currentRelayActivity(messages) : null);
+    }, [bridge, messages, status, running]);
     useEffect(() => {
         const last = messages[messages.length - 1];
         if (!last || last.role !== "assistant")

package/dist/react/voice-stage.js

@@ -35,6 +35,7 @@ export function VoiceStage(props = {}) {
     const connectionState = useVoiceUiStore((s) => s.connectionState);
     const userText = useVoiceUiStore((s) => s.userText);
     const assistantText = useVoiceUiStore((s) => s.assistantText);
+    const transcriptRole = useVoiceUiStore((s) => s.transcriptRole);
     const chromaKey = useVoiceUiStore((s) => s.chromaKey);
     const connecting = !voice || voice.status.type === "starting";
     const muted = voice?.isMuted ?? false;
@@ -49,9 +50,8 @@ export function VoiceStage(props = {}) {
                     : sessionState === "speaking"
                         ? "Speaking"
                         : "Listening";
-    // While the assistant talks the visualiser is showing its voice, so the
-    // caption should be its words; otherwise show what was just heard.
-    const caption = sessionState === "speaking" ? assistantText : userText;
+    // Pauses in playback do not change who last spoke.
+    const caption = transcriptRole === "assistant" ? assistantText : userText;
     return (_jsxs("div", { className: "relative flex h-full min-h-full flex-col", children: [_jsx("div", { className: "relative min-h-0 flex-1 overflow-hidden", children: videoStream ? (
                 // Full-bleed and anchored to the top, so the avatar is framed head
                 // and torso with the legs running off the lower edge — the framing

package/dist/react/voice-surface.js

@@ -26,7 +26,7 @@ import { VoiceStage } from "./voice-stage.js";
 import { needsAudioOnlySink } from "./audio-ownership.js";
 import { VoiceAudioSink } from "./voice-audio-sink.js";
 import { useDirectAudioOutput } from "./use-direct-audio-output.js";
-import { resetVoiceSessionState, useVoiceUiStore } from "./voice-ui-store.js";
+import { resetVoiceSessionState, updateVoiceCaption, useVoiceUiStore } from "./voice-ui-store.js";
 export function VoiceSurface({ bridge, config, avatar = true, stage = true }) {
     const mode = config.mode ?? "relay";
     const wantsTools = mode === "realtime" && (config.exposeClientTools ?? true);
@@ -38,11 +38,7 @@ export function VoiceSurface({ bridge, config, avatar = true, stage = true }) {
     const onEvent = useCallback((event) => bridge.handleServerEvent(event), [bridge]);
     const onTranscript = useCallback((role, text, isFinal) => {
         bridge.handleTranscript(role, text, isFinal);
-        // The stage shows what is being said right now; a final empty string
-        // is a VAD misfire and would blank the line for no reason.
-        if (text.trim() || !isFinal) {
-            useVoiceUiStore.setState(role === "user" ? { userText: text } : { assistantText: text });
-        }
+        updateVoiceCaption(role, text);
     }, [bridge]);
     const toolExecutor = useMemo(() => (wantsTools ? createRegistryToolExecutor(clientToolRegistry.getState()) : undefined), [wantsTools]);
     const live = useVoiceLive({

package/dist/react/voice-ui-store.d.ts

@@ -30,6 +30,8 @@ interface VoiceUiState {
      */
     userText: string;
     assistantText: string;
+    /** Latest non-empty transcript's speaker; independent of playback pauses. */
+    transcriptRole: "user" | "assistant" | null;
     /** Avatar streams, when the session was configured with one. */
     videoStream: MediaStream | null;
     audioStream: MediaStream | null;
@@ -53,6 +55,8 @@ export declare const useVoiceUiStore: import("zustand").UseBoundStore<import("zu
  * definition, and visibly so on the stage.
  */
 export declare function resetVoiceSessionState(): void;
+/** Keep the last caption until somebody actually says something new. */
+export declare function updateVoiceCaption(role: "user" | "assistant", text: string): void;
 /** Called by `VoiceAvatarPanel` as it mounts and unmounts. */
 export declare function trackAvatarPanel(delta: 1 | -1): void;
 export {};

package/dist/react/voice-ui-store.js

@@ -21,6 +21,7 @@ export const useVoiceUiStore = create(() => ({
     analyser: null,
     userText: "",
     assistantText: "",
+    transcriptRole: null,
     videoStream: null,
     audioStream: null,
     chromaKey: null,
@@ -35,11 +36,21 @@ export function resetVoiceSessionState() {
     useVoiceUiStore.setState({
         userText: "",
         assistantText: "",
+        transcriptRole: null,
         analyser: null,
         videoStream: null,
         audioStream: null,
     });
 }
+/** Keep the last caption until somebody actually says something new. */
+export function updateVoiceCaption(role, text) {
+    if (!text.trim())
+        return;
+    useVoiceUiStore.setState({
+        transcriptRole: role,
+        ...(role === "user" ? { userText: text } : { assistantText: text }),
+    });
+}
 /** Called by `VoiceAvatarPanel` as it mounts and unmounts. */
 export function trackAvatarPanel(delta) {
     useVoiceUiStore.setState((s) => ({ avatarPanels: Math.max(0, s.avatarPanels + delta) }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iloveagents/foundry-web-voice",
-  "version": "0.1.12",
+  "version": "0.1.14",
   "description": "Optional spoken-conversation tier for Foundry UI — Azure Voice Live wired into assistant-ui's realtime voice contract",
   "keywords": [
     "foundry",
@@ -38,8 +38,8 @@
   ],
   "dependencies": {
     "@iloveagents/foundry-voice-live-react": "^0.5.0",
-    "@iloveagents/foundry-agent": "^0.32.0",
-    "@iloveagents/foundry-web-ui": "^0.32.0"
+    "@iloveagents/foundry-agent": "^0.33.0",
+    "@iloveagents/foundry-web-ui": "^0.33.0"
   },
   "peerDependencies": {
     "@assistant-ui/react": "^0.15.1",