@athenaintel/react 0.41.2 → 0.42.1

package/dist/agent-runtime/src/use-deep-agent-voice.d.ts

@@ -0,0 +1,409 @@
+import type { DeepAgentVoiceWireError } from './voice-wire-types';
+declare module '@assistant-ui/core' {
+    namespace Assistant {
+        interface Commands {
+            /**
+             * Opens a voice session (`DeepAgentThread.voice_start`). `offer` is the
+             * WebRTC SDP offer, {@link VOICE_WEBSOCKET_OFFER} to request the
+             * WebSocket audio transport (the answer is then a relay ticket), or
+             * {@link VOICE_LIVEKIT_OFFER} to request a LiveKit room (the answer is
+             * then the room's join credentials). `model` picks the realtime model
+             * (`gpt-live-1`, `gemini-3.8-live`, …); omitted → the host's default.
+             */
+            voiceStart: {
+                type: 'voice/start';
+                offer: string;
+                model?: string;
+            };
+            /** Ends the live voice session (`DeepAgentThread.voice_end`). */
+            voiceEnd: {
+                type: 'voice/end';
+            };
+            /**
+             * No-op whose INGRESS refreshes the caller's server-stored context
+             * (context syncs ride command sends), so runConfig changes made while a
+             * voice session is live reach the delegated runs.
+             */
+            voiceProfileSync: {
+                type: 'voice/profile-sync';
+            };
+            /**
+             * Mute/unmute the live session's microphone at the protocol level
+             * (`DeepAgentThread.voice_set_mic` → `session.input_audio.mute`/`unmute`).
+             */
+            voiceSetMic: {
+                type: 'voice/set-mic';
+                paused: boolean;
+            };
+            /**
+             * Route a typed message into the live conversation
+             * (`DeepAgentThread.voice_user_text`): commits durably as a
+             * `source: "text"` user turn and reaches the live model as speakable
+             * context, so it responds by voice exactly as it would to speech.
+             * `messageId` is client-minted so a durable-command redelivery after a
+             * host crash commits idempotently instead of duplicating the turn.
+             */
+            voiceUserText: {
+                type: 'voice/user-text';
+                text: string;
+                messageId: string;
+            };
+            /**
+             * Tell the live session what the user is looking at
+             * (`DeepAgentThread.voice_context`): one short description of the open
+             * asset or page, appended as silent commentary so the live model can
+             * resolve "this document" without delegating. Never spoken, never
+             * persisted; identical consecutive descriptions are dropped server-side.
+             */
+            voiceContext: {
+                type: 'voice/context';
+                text: string;
+            };
+        }
+    }
+}
+/** A voice-delegated frontend tool call projected by legacy hosts
+ * (`arguments` is a JSON string). Current hosts run frontend tools through
+ * the delegated run's client-tool interrupts, so this stays empty. */
+export interface DeepAgentVoiceToolCall {
+    callId: string;
+    name: string;
+    arguments: string;
+}
+/** The host's `state.voice` projection while a session exists. */
+export interface DeepAgentVoiceWireState {
+    clientId: string;
+    status: 'connecting' | 'live';
+    toolCalls: DeepAgentVoiceToolCall[];
+    /** The delegated voice task currently running, if any. */
+    activeTask?: {
+        taskId: string;
+        text: string;
+    } | null;
+}
+export type DeepAgentVoicePhase = 'idle' | 'connecting' | 'live' | 'error';
+/**
+ * Why a voice session failed, classified where the failure happened rather
+ * than by parsing the message afterwards. `server_rejected` carries the
+ * host's structured code (`voice-active`, `voice-not-enabled`,
+ * `voice-unavailable`, `checkpointer_unavailable`, …); `media_lost` carries the
+ * transport state that ended the session. Every surface reports and words the
+ * same kinds, so telemetry across Olympus, Chrome, and mobile joins on them.
+ */
+export type DeepAgentVoiceFailureKind = 'unsupported' | 'audio_session' | 'microphone_denied' | 'capture_failed' | 'offer_failed' | 'answer_timeout' | 'server_rejected' | 'command_failed' | 'no_answer' | 'answer_failed' | 'relay_failed' | 'room_failed' | 'playback_failed' | 'media_lost' | 'unknown';
+/** Where in the session's lifecycle a failure happened. */
+export type DeepAgentVoiceFailureStage = 'preflight' | 'audio-session' | 'capture' | 'offer' | 'negotiate' | 'answer' | 'relay' | 'room' | 'media' | 'playback';
+export interface DeepAgentVoiceFailure {
+    kind: DeepAgentVoiceFailureKind;
+    stage: DeepAgentVoiceFailureStage;
+    /** The message surfaces render (the same value as `error`). */
+    message: string;
+    /** The host's rejection reason / structured `detail.code`, when it sent one. */
+    code?: string;
+    /** The rejection's HTTP-style status, when the host sent a structured one. */
+    status?: number;
+    /** The statewire send fate when the command settled without a verdict. */
+    fate?: string;
+    /**
+     * The lower-level reason: the media state that ended the session
+     * (`failed` / `closed` / `disconnected` / `relay-closed` / `relay-error`), or
+     * the channel death that settled the command (`unmount` / `stream-gone` /
+     * `session-lost`).
+     */
+    reason?: string;
+}
+/**
+ * Lifecycle milestones of a session attempt, for surfaces that record them
+ * (crash breadcrumbs, latency telemetry). `attempt` groups one start's
+ * events; `sinceStartMs` is measured from that attempt's `start-requested`.
+ */
+export type DeepAgentVoiceDiagnosticEvent = 'start-requested' | 'audio-session-ready' | 'microphone-captured' | 'offer-ready' | 'ice-gathering-timeout' | 'answer-received' | 'room-joined' | 'live' | 'media-state' | 'media-loss' | 'auto-restart' | 'server-ended' | 'end' | 'error' | 'stale-attempt';
+export interface DeepAgentVoiceDiagnostic {
+    event: DeepAgentVoiceDiagnosticEvent;
+    attempt: number;
+    /** Epoch ms. */
+    at: number;
+    sinceStartMs: number;
+    data?: Record<string, string | number | boolean | null>;
+}
+/**
+ * Classify a failed `voice/start` command. A host rejection keeps the host's
+ * reason (`payload.reason` from the voice manager, `payload.detail.code` from
+ * the command gate's structured envelope) and its own user-worded message; a
+ * command the channel settled without a verdict reports its fate.
+ */
+export declare const classifyVoiceStartError: (error: unknown) => Pick<DeepAgentVoiceFailure, 'kind' | 'message' | 'code' | 'status' | 'fate' | 'reason'>;
+/**
+ * Decode a `voice/start` answer on the `livekit` transport. Throws (as a
+ * `no_answer` failure when raised in the `answer` stage) on a missing prefix,
+ * malformed JSON, or a missing/non-string field — a host that answered with
+ * something else did not open a LiveKit room for this client.
+ */
+export declare const parseVoiceLiveKitAnswer: (answer: string) => DeepAgentVoiceLiveKitAnswer;
+/**
+ * User-facing copy for a failure, shared so every surface words the same
+ * cause alike. Host rejections keep the host's own wording (it already names
+ * the actionable cause); the local kinds get a sentence the user can act on.
+ */
+export declare const describeVoiceFailure: (failure: DeepAgentVoiceFailure) => string;
+/**
+ * How the session's audio travels. `webrtc` (the default) negotiates a peer
+ * connection straight to OpenAI; `websocket` captures PCM locally and streams
+ * it to Agora's relay endpoint, for surfaces where WebRTC is unavailable
+ * (Office add-in webviews, UDP-blocked networks) — web only: it needs
+ * `AudioContext` and `WebSocket` globals. `livekit` joins the LiveKit room the
+ * host mints for the session through a caller-supplied
+ * {@link DeepAgentVoiceLiveKitAdapter}; Agora joins the same room as the agent
+ * participant, so this is the only transport on which the realtime model is a
+ * per-session choice (`gpt-live-1`, `gemini-3.8-live`, …) — the other two are
+ * OpenAI-direct media legs and stay GPT Live only.
+ */
+export type DeepAgentVoiceTransport = 'webrtc' | 'websocket' | 'livekit';
+/**
+ * The `voice/start` offer that requests the WebSocket transport. Mirrors
+ * `agora.services.voice_relay.VOICE_WEBSOCKET_OFFER`.
+ */
+export declare const VOICE_WEBSOCKET_OFFER = "athena-voice-transport:websocket/v1";
+/** Prefix of a `voice/start` answer that carries a relay ticket, not an SDP answer. */
+export declare const VOICE_RELAY_ANSWER_PREFIX = "athena-voice-relay:";
+/** The relay's wire format: raw PCM16, mono, 24 kHz (the Live API's PCM format). */
+export declare const VOICE_RELAY_SAMPLE_RATE = 24000;
+/**
+ * The `voice/start` offer that requests the LiveKit transport. Mirrors
+ * `agora.graphs.athena_deep_agent.voice_media.VOICE_LIVEKIT_OFFER`.
+ */
+export declare const VOICE_LIVEKIT_OFFER = "athena-voice-transport:livekit/v1";
+/**
+ * Prefix of a `voice/start` answer that carries LiveKit room credentials: the
+ * rest is `JSON.stringify({ serverUrl, token, roomName })`. Mirrors
+ * `agora.graphs.athena_deep_agent.voice_media.VOICE_LIVEKIT_ANSWER_PREFIX`.
+ */
+export declare const VOICE_LIVEKIT_ANSWER_PREFIX = "athena-voice-livekit:";
+/** The room the host minted for the session, decoded from the `voice/start` answer. */
+export interface DeepAgentVoiceLiveKitAnswer {
+    /** LiveKit signalling URL, already `wss://`. */
+    serverUrl: string;
+    /** The browser participant's join token (`room_join`, publish, subscribe). */
+    token: string;
+    roomName: string;
+}
+/** A joined LiveKit room, as far as the session state machine drives it. */
+export interface DeepAgentVoiceLiveKitSession {
+    /** Publish/unpublish (or unmute/mute) the local microphone track. */
+    setMicrophoneEnabled(enabled: boolean): Promise<void>;
+    /** Leave the room and release the microphone. Idempotent. */
+    disconnect(): Promise<void>;
+}
+/**
+ * The LiveKit leg of the `livekit` transport, supplied by the surface so this
+ * package never depends on `livekit-client` (web surfaces wrap `Room`; native
+ * ones wrap the React Native SDK). Structural on purpose: tests inject fakes.
+ */
+export interface DeepAgentVoiceLiveKitAdapter {
+    /**
+     * Join the room, publish the microphone (respecting `micMuted`), play remote
+     * audio; rejects on failure (a microphone-permission error keeps its DOM
+     * `name`, so it classifies as `microphone_denied`). `onDisconnected` fires
+     * once when the room drops AFTER a successful join — never for the session's
+     * own `disconnect()`. An aborted `signal` must stop pending/joined media and
+     * reject a pending join; the runtime also releases late results from older
+     * adapters that do not implement cancellation.
+     */
+    connect(answer: DeepAgentVoiceLiveKitAnswer, handlers: {
+        micMuted: boolean;
+        /** Abort a pending join and release media when this attempt ends. */
+        signal?: AbortSignal;
+        onDisconnected: (reason: string) => void;
+    }): Promise<DeepAgentVoiceLiveKitSession>;
+}
+/** One realtime model the host offers for voice sessions (`state.voiceModelOptions`). */
+export interface DeepAgentVoiceModelOption {
+    id: string;
+    label: string;
+    provider: 'openai_live' | 'google';
+    /** `false` when the deployment lacks the provider's credentials. */
+    enabled: boolean;
+    description: string;
+}
+/**
+ * Validate the host's `state.voiceModelOptions`: `null` when absent or not a
+ * list, otherwise the well-formed entries (a malformed entry is dropped, not
+ * fatal — an older or newer host must never blank the whole picker).
+ */
+export declare const parseVoiceModelOptions: (value: unknown) => DeepAgentVoiceModelOption[] | null;
+interface VoiceMediaStreamTrack {
+    /** Standard WebRTC mute lever: a disabled track transmits silence. */
+    enabled: boolean;
+    stop(): void;
+}
+export interface DeepAgentVoiceRemoteStream {
+    getTracks(): VoiceMediaStreamTrack[];
+    /** React Native WebRTC owns native stream/track entries beyond track.stop(). */
+    release?(): void;
+}
+/** `m:ss` / `h:mm:ss` for the live-session timer strips (shared so the three
+ * surfaces cannot drift on formatting). */
+export declare const formatVoiceElapsed: (sinceMs: number, nowMs?: number) => string;
+/** Ticking elapsed-call string for a live session — the one timer
+ * implementation all surfaces share (interval lifecycle + formatting).
+ * Returns `null` when there is no live session. Callers isolate it in a
+ * leaf component so only that leaf re-renders per second. */
+export declare const useVoiceElapsed: (since: number | null) => string | null;
+/**
+ * One-line, user-facing rendering of a Live API rejection the host projected
+ * (`state.voiceLastError`). Shared so every surface words the notice alike.
+ */
+export declare const describeVoiceWireError: (error: DeepAgentVoiceWireError) => string;
+/** The relay endpoint for a ticket: `wss://<agora>/api/voice/relay?ticket=…`. */
+export declare const voiceRelayUrl: (relayUrl: string, ticket: string) => string;
+/** Float32 samples in [-1, 1] → little-endian PCM16 (the relay wire format). */
+export declare const pcm16FromFloat32: (samples: Float32Array) => Int16Array;
+/** PCM16 → Float32 samples in [-1, 1] for Web Audio playback. */
+export declare const float32FromPcm16: (pcm: Int16Array) => Float32Array;
+export interface UseDeepAgentVoiceOptions {
+    /**
+     * Remote-audio sink: called with the remote stream when the session answers
+     * and with `null` on teardown. Web surfaces attach it to an `<audio>`
+     * element; react-native-webrtc plays remote audio natively, so native
+     * surfaces omit it. A rejected promise fails the session. WebRTC only: the
+     * WebSocket transport plays through its own `AudioContext`.
+     */
+    onRemoteStream?: (stream: DeepAgentVoiceRemoteStream | null) => void | Promise<void>;
+    /**
+     * Platform audio-session ownership for the whole call: `start` runs before
+     * mic capture (a rejection fails the session), `stop` runs on every
+     * teardown path. Native surfaces pass their call-audio module (on iOS the
+     * non-mixable playAndRecord/voiceChat session that entitles the app to
+     * keep a live call running); web surfaces omit it.
+     */
+    audioSession?: {
+        start: () => Promise<void>;
+        stop: () => Promise<void>;
+        /** Synchronously mark an in-flight start for cleanup before a new owner starts. */
+        cancelStart?: () => void;
+    };
+    /** Media transport; see {@link DeepAgentVoiceTransport}. Default `webrtc`. */
+    transport?: DeepAgentVoiceTransport;
+    /**
+     * Agora's base URL (e.g. `https://api.athenaintel.com`) whose
+     * `/api/voice/relay` endpoint carries the audio on the `websocket`
+     * transport. Required for that transport, ignored otherwise.
+     */
+    relayUrl?: string;
+    /**
+     * The room leg for the `livekit` transport (required for it, ignored
+     * otherwise); see {@link DeepAgentVoiceLiveKitAdapter}. Read through a
+     * per-render ref, so a fresh adapter never restarts anything.
+     */
+    livekit?: DeepAgentVoiceLiveKitAdapter;
+    /**
+     * The realtime model to ask for, sent as `voice/start.model` — a value or a
+     * getter resolved when each attempt (user start or automatic restart)
+     * negotiates. `undefined`/empty omits the key and the host picks its
+     * default. Only the `livekit` transport can honour a non-GPT-Live model.
+     */
+    model?: string | (() => string | undefined);
+    /**
+     * Lifecycle observer for the session attempts (negotiation milestones with
+     * timings, media-state changes, losses, restarts, ends). Surfaces feed it to
+     * their crash breadcrumbs / telemetry; read through a per-render ref, so a
+     * fresh closure never restarts anything. Must not throw.
+     */
+    onDiagnostic?: (entry: DeepAgentVoiceDiagnostic) => void;
+    /** Per-surface overrides of the negotiation deadlines (ms). */
+    timeouts?: {
+        /** How long to wait for ICE gathering to complete before sending the offer with the candidates so far. */
+        iceGatheringMs?: number;
+        /** How long `voice/start` may take to answer. */
+        answerMs?: number;
+        /** How long the relay WebSocket may take to open (`websocket` transport). */
+        relayOpenMs?: number;
+    };
+}
+export interface UseDeepAgentVoiceReturn {
+    phase: DeepAgentVoicePhase;
+    /** The failure message while `phase === 'error'` (same as `failure.message`). */
+    error: string | null;
+    /**
+     * The classified failure while `phase === 'error'`; `null` otherwise. Where
+     * `error` is only words, this says WHICH step failed and, for a host
+     * rejection, the host's reason code — the key telemetry and support join on.
+     */
+    failure: DeepAgentVoiceFailure | null;
+    start: () => Promise<void>;
+    end: () => void;
+    /** The delegated voice task currently running server-side, if any. */
+    activeTask: {
+        taskId: string;
+        text: string;
+    } | null;
+    /** Epoch ms when the session went live; `null` outside a live session.
+     * Surfaces render an elapsed-call timer from it. */
+    liveSince: number | null;
+    /**
+     * Whether the local microphone is muted. Mute is distinct from `end`: the
+     * session (and any delegated work) stays live while the local track is
+     * disabled AND the server mutes protocol input (`session.input_audio.mute`
+     * — muted speech is discarded server-side, the model keeps speaking).
+     */
+    micMuted: boolean;
+    /** Mute/unmute the live session's microphone; a no-op unless live. */
+    setMicMuted: (muted: boolean) => void;
+    /**
+     * Bimodal input: send typed text into the live conversation as if spoken.
+     * Resolves `true` once the server accepted it; `false` when there is no
+     * live session or the command was rejected — the caller falls back to a
+     * normal run send so the message is never lost.
+     */
+    sendUserText: (text: string) => Promise<boolean>;
+    /**
+     * The delegated task that was still running when the last session ended.
+     * The run itself continues on the thread — surfaces render a "voice ended,
+     * still working" note from this until the run settles or a new session
+     * starts, at which point it clears itself.
+     */
+    handoffTask: {
+        taskId: string;
+        text: string;
+    } | null;
+    /**
+     * Push the client's current synced context (runConfig included) to the
+     * server so the NEXT delegated run uses it. Call after a runConfig change
+     * while the session is live; a no-op otherwise.
+     */
+    syncProfile: () => void;
+    /**
+     * Non-fatal notice for the live session: the most recent Live API
+     * rejection the host projected (a refused context update, a failed mute),
+     * worded for the user. The session stays live; `null` when there is none
+     * or the session is not live.
+     */
+    notice: string | null;
+    /**
+     * Tell the live session what the user is looking at (one short line, e.g.
+     * `spreadsheet "Q3 Budget"`). Silent context for reference resolution;
+     * fire-and-forget, a no-op unless live.
+     */
+    sendContext: (text: string) => void;
+    /**
+     * The realtime model the host resolved for the current/last session
+     * (`state.voiceModel`, stamped when a session starts), or `null` before
+     * any session or while the snapshot is not hydrated. Stale after an end is
+     * inert — surfaces gate on `phase`, like `micMuted`.
+     */
+    model: string | null;
+    /**
+     * The realtime models this host offers (`state.voiceModelOptions`,
+     * published at host boot), validated; `null` on an older host that does
+     * not publish them — surfaces fall back to their static list.
+     */
+    modelOptions: DeepAgentVoiceModelOption[] | null;
+}
+/**
+ * The experimental voice-mode session against the thread's `voice/start` /
+ * `voice/end` commands: mic capture and media negotiation (WebRTC by default,
+ * or PCM over Agora's relay WebSocket); the live turns stream into the chat
+ * log as `ephemeral: true` entries in `state.messages`.
+ */
+export declare const useDeepAgentVoice: (options?: UseDeepAgentVoiceOptions) => UseDeepAgentVoiceReturn;
+export {};