@iloveagents/foundry-web-voice 0.1.0

package/dist/adapter/voice-bridge.d.ts

@@ -0,0 +1,214 @@
+/**
+ * `VoiceBridge` — everything the voice tier does that is not React.
+ *
+ * It sits between two halves that cannot see each other:
+ *
+ *   - **assistant-ui**, which asks for a `RealtimeVoiceAdapter`: an
+ *     imperative `connect()` returning a session that emits transcripts,
+ *     mode, volume and status.
+ *   - **the Voice Live SDK**, whose entry point is the `useVoiceLive` hook —
+ *     a React object that only exists while a component is mounted.
+ *
+ * The join is `createVoiceSession(options, setup)`, whose `setup` is
+ * **async**: `connect()` can return a session in the `starting` state
+ * immediately and resolve the transport afterwards. So a render-less host
+ * component owns the hook and calls `attach()` with its controls, and the
+ * bridge's `connect()` simply awaits them. That reuses the SDK whole —
+ * WebRTC, avatar, reconnect, the response gate, the tool round-trip —
+ * instead of reimplementing a transport to satisfy an interface shape.
+ *
+ * The bridge holds no React component, hook or rendered state — the only
+ * thing it takes from `@assistant-ui/react` is the voice contract itself
+ * (`createVoiceSession`, the adapter types). So every rule below is driven
+ * and asserted as a plain object against a fake transport, with no tree to
+ * render and no session to connect.
+ */
+import { type RealtimeVoiceAdapter } from "@assistant-ui/react";
+import type { VoiceLiveClientEvent, VoiceLiveServerEvent, SessionState } from "@iloveagents/foundry-voice-live-react";
+import type { VoiceConfig } from "./types.ts";
+/** What the bridge needs from the live session; supplied by the React host. */
+export interface VoiceTransportControls {
+    connect: () => Promise<void>;
+    disconnect: () => void;
+    mute: () => void;
+    unmute: () => void;
+    /** Put a client event on the wire. `response.create` is serialized by the SDK's response gate. */
+    sendEvent: (event: VoiceLiveClientEvent) => void;
+    /** Cancel the in-flight response and flush local playback (barge-in). */
+    cancelResponse: () => void;
+    /** Output analyser for a level meter, when the session exposes one. */
+    getAnalyser: () => AnalyserNode | null;
+}
+type Unsubscribe = () => void;
+export declare class VoiceBridge {
+    private readonly config;
+    private readonly mode;
+    private controls;
+    private waiters;
+    private helpers;
+    private connected;
+    /**
+     * The user's mute intent, as expressed through the runtime's controls.
+     *
+     * Distinct from the SDK's capture-mute state on purpose: half-duplex mutes
+     * the SDK on its own behalf while the assistant speaks, and releasing that
+     * mute must never override a mute the *user* asked for — that is how a
+     * microphone ends up hot behind a UI that says muted.
+     */
+    private userMuted;
+    /** An answer that was already on screen when voice connected — not ours to speak. */
+    private skipAnswerId;
+    private volumeTimer;
+    private volumeBuffer;
+    private readonly speech;
+    private readonly sentences;
+    /** Id of the assistant message currently being spoken, so a new answer resets the stream. */
+    private answerId;
+    /**
+     * Whether `answerId` has been spoken to the end.
+     *
+     * The watcher re-fires for an answer that is already finished — the
+     * thread's message array gets a new identity on any notification, not just
+     * on new text. Without a "done" marker, clearing `answerId` on completion
+     * made the very next tick look like a brand-new answer, reset the sentence
+     * stream, and speak the whole reply again. Forever.
+     */
+    private answerDone;
+    /** The response currently being spoken, once the service has named it. */
+    private activeResponseId;
+    /**
+     * Responses cancelled by barge-in whose `response.done` has not arrived yet.
+     *
+     * Bounded by the number of interruptions in a call and cleared with it. A
+     * cancelled response that never reports simply leaves an unused entry — the
+     * alternative, counting completions to swallow, wedges the queue for the
+     * rest of the session when the count is wrong.
+     */
+    private readonly cancelledResponses;
+    /**
+     * A response was cancelled before the service had named it.
+     *
+     * The SDK's own gate documents the window: between sending `response.create`
+     * and receiving `response.created` the conversation looks idle. Interrupting
+     * in there leaves nothing to correlate, so the completion that eventually
+     * arrives would look like the NEXT answer's and release it early — the same
+     * bug, in the gap. The wire preserves order, so the first `response.created`
+     * after such a cancel names the response that was cancelled.
+     */
+    private cancelledBeforeCreated;
+    /**
+     * The connection attempt currently allowed to become a session.
+     *
+     * A token per attempt, not a boolean on the bridge. A boolean is shared, so
+     * starting a second call reset it — and if the FIRST transport then came up,
+     * its continuation saw "not abandoned", marked itself connected, and left a
+     * microphone open with no host, while also clearing the newer attempt's
+     * pending record. Each attempt compares its own token, so an obsolete
+     * continuation can never mistake a later call's state for permission.
+     */
+    private attempt;
+    /**
+     * A connection that has been asked for but has not settled.
+     *
+     * Held so that abandoning a call can take the transport down NOW rather than
+     * waiting on a promise that may never resolve — a permission prompt left
+     * open, a transport that stalls. `setup` re-checks its attempt token past
+     * the await and covers a connection that arrives late; the case it cannot reach
+     * is the one that never arrives, which is exactly this.
+     */
+    private pendingConnect;
+    /** Injected so the bridge never imports the React chat surface. */
+    private submitText;
+    constructor(config: VoiceConfig);
+    /**
+     * True only once the transport is actually carrying a session.
+     *
+     * `helpers` exists from the first line of `setup`, which is *before*
+     * `controls.connect()` resolves. Speaking in that window sends into a
+     * socket that is not open yet: the utterance is lost, and because the
+     * queue is now "speaking" it never receives the `response.done` that
+     * would release it — the session goes silent for good.
+     */
+    private get isLive();
+    /** The user's mute intent (see `userMuted`). Read by the half-duplex gate. */
+    get isUserMuted(): boolean;
+    /** The React host publishes its live controls here (and retracts them on unmount). */
+    attach(controls: VoiceTransportControls): Unsubscribe;
+    /** How the transcript reaches the composer in `relay` mode. */
+    setSubmitText(submit: ((text: string) => void) | null): void;
+    createAdapter(): RealtimeVoiceAdapter;
+    /**
+     * A transcript line from the SDK (`text` is cumulative for the turn).
+     *
+     * The modes diverge here, and the divergence is the whole reason a spoken
+     * turn in `relay` renders as an ordinary, persisted thread turn:
+     *
+     * - `realtime` republishes both roles, and assistant-ui materialises them
+     *   as (session-scoped) thread messages.
+     * - `relay` republishes **neither**. The user's words become a real
+     *   message by going through the composer, and the answer is a real AG-UI
+     *   turn. Emitting the transcript as well would render each of them
+     *   twice.
+     */
+    handleTranscript(role: "user" | "assistant", text: string, isFinal: boolean): void;
+    /** Raw server events — only the few that drive turn-taking are read. */
+    handleServerEvent(event: VoiceLiveServerEvent): void;
+    /** SDK session state → assistant-ui's two-value mode. */
+    handleSessionState(state: SessionState): void;
+    /** A session-fatal error: report it, and end the session so the UI leaves the live state. */
+    handleError(message: string): void;
+    /**
+     * The transport reached a terminal closed state.
+     *
+     * Idempotent by the `connected` check, because every route out of a call
+     * ends here: an explicit `disconnect()`, a failure through `handleError`,
+     * and the surface observing `connectionState === "disconnected"` all arrive,
+     * and ending the session twice would be a second `end()` on a session that
+     * already finished.
+     */
+    handleClosed(): void;
+    /**
+     * Everything a finished call must let go of, whichever way it ended.
+     *
+     * This bridge is installed once for the life of the app, so its state
+     * outlives any single call. The speech queue is the dangerous part: it
+     * releases on `response.done`, and a transport that has died will never send
+     * one. Left `speaking`, it swallows every utterance of every LATER call —
+     * the session connects, the agent answers, and nothing is ever heard again.
+     *
+     * `disconnect()` always did this; an error or a remote close did not, which
+     * is exactly the pair of routes on which the transport dies mid-utterance.
+     */
+    private endCall;
+    /**
+     * Track the assistant answer as it streams and speak it sentence by
+     * sentence. `relay` only — in `realtime` the model is already speaking.
+     *
+     * @param messageId identity of the answer; a change resets the stream
+     * @param text cumulative answer text so far (markdown)
+     * @param isComplete the turn has settled
+     */
+    /**
+     * Mark an answer as already-delivered, so connecting voice does not read
+     * the last thing on screen back to the user. Called by the relay watcher
+     * the moment the session goes live.
+     */
+    primeAnswer(messageId: string | null): void;
+    trackAnswer(messageId: string, text: string, isComplete: boolean): void;
+    /**
+     * Speak exact text through Voice Live without invoking its model.
+     *
+     * `pre_generated_assistant_message` is the documented way to do this, and
+     * it is what keeps the avatar's lip-sync and the chosen Azure voice while
+     * the words come from somewhere else entirely. Sent through the SDK's
+     * `sendEvent`, which routes a raw `response.create` through the same
+     * response gate as every other turn.
+     */
+    private sendSpokenText;
+    private awaitControls;
+    private startVolumeMeter;
+    private ensureVolumeBuffer;
+    private stopVolumeMeter;
+}
+export declare function readLevel(analyser: AnalyserNode | null, getBuffer: (size: number) => Uint8Array<ArrayBuffer>): number;
+export {};