@theoremjs/react 0.0.0-stage → 0.3.1

package/client/live-client.d.ts

@@ -0,0 +1,189 @@
+/**
+ * Pure client-side SDK for THEOREM Gemini 3.1 Flash Live sessions over WebSocket relay.
+ *
+ * Handles:
+ * - Bidirectional WebSocket connection to `/api/live/relay`.
+ * - 16-bit 16kHz PCM microphone audio recording & streaming.
+ * - Gapless 24kHz PCM / WAV model voice playback scheduling.
+ * - Barge-in interruption cancellation (instant audio queue flush).
+ * - Quiet-mic gate while model audio plays (reduces false barge-in from speaker bleed).
+ * - UI tool execution and response routing.
+ *
+ * @module
+ */
+import { type SessionEventOf, type TraceRecord } from '@theoremjs/agents';
+import type { LiveConnectPhase, LiveSessionStatus } from './live/live-state.ts';
+import { type ExecuteToolOnRelay, type LiveToolStep } from './live-messages.ts';
+import { type ClientTurnEvent } from './transport.ts';
+export type { LiveConnection, LiveSocket } from './live-messages.ts';
+import type { LiveSocket } from './live-messages.ts';
+export interface LiveClientOptions {
+    profile?: string;
+    /** The host's own JSON for its relay, sent as the open message's `host`. */
+    openMessage?: Record<string, unknown>;
+    /** The value chosen for each of the profile's `inputs.slots`, fixed for the call. */
+    slots?: Record<string, string>;
+    /** What the page tells the agent as the call opens; `setContext` replaces it during the call. */
+    context?: unknown;
+    relayUrl?: string;
+    /** In-process connections reuse the same microphone, playback, and tool client. */
+    createSocket?: () => LiveSocket;
+    /** When false, skip microphone capture; session still receives model audio. */
+    voiceIngress?: boolean;
+    onStatusChange?: (status: LiveSessionStatus) => void;
+    onConnectPhase?: (phase: LiveConnectPhase | null) => void;
+    onTranscript?: (text: string, isUser: boolean, meta?: {
+        interim?: boolean;
+    }) => void;
+    /**
+     * Every event the session sends, `unsupported` for a kind this client does
+     * not know, and `malformed` for one that failed its check (left out; the call goes on).
+     */
+    onTurnEvent?: (event: ClientTurnEvent) => void;
+    /** A trace record the session wrote, when the relay delivers them. */
+    onTrace?: (record: TraceRecord) => void;
+    /** A failure, typed by kind; word it with `clientFailure` and the interface's `labels`. */
+    onError?: (error: Error) => void;
+    /** Provider signalled the upstream session is draining (e.g. goAway). */
+    onSessionClosing?: (timeLeftMs?: number) => void;
+    /**
+     * The provider ended the session after warning it would: not a failure.
+     * `session.message` is the user's line; `session.ended` the close, for the builder.
+     */
+    onSessionEnded?: (session: SessionEventOf<'ended'>) => void;
+    /**
+     * The model called a tool. The host runs it with `executeToolOnRelay` and
+     * answers its gates; the session answers the model. A throw reaches `onError`.
+     */
+    onToolCall: (name: string, args: Record<string, unknown>, meta: {
+        callId: string;
+        operationId: string;
+    }) => Promise<void>;
+    /** The relay opened the call: the id it gave the call, and the profile it runs. */
+    onSessionReady?: (info: {
+        sessionId?: string;
+        profile?: string;
+    }) => void;
+    onVolumeLevel?: (level: number, isUser: boolean) => void;
+}
+export declare class LiveSessionClient {
+    private ws;
+    private audioContext;
+    private micStream;
+    private micSource;
+    private micWorklet;
+    private micWorkletModuleLoaded;
+    private playbackNodes;
+    private playbackBus;
+    private playbackAnalyser;
+    private playbackMeterFrame;
+    private playbackMeterBuffer;
+    private nextPlaybackTime;
+    private status;
+    /** Server reported background work (reasoning / async tools) still in flight. */
+    private serverWorking;
+    private micActivating;
+    private connectTimeout;
+    private isMuted;
+    private options;
+    /** Serialize control-plane handling (tools / status) without waiting on audio decode. */
+    private inboundChain;
+    /** Ordered model-audio playback queue (separate so tools are not stuck behind decode). */
+    private audioChain;
+    /**
+     * Tool calls, one at a time in the order the model made them. Apart from
+     * `inboundChain`: a call waits on the relay's reply, which arrives inbound.
+     */
+    private toolChain;
+    /** Bumped on barge-in / cancel so stale audioChain work is skipped. */
+    private audioEpoch;
+    private pendingExecuteResults;
+    /** The page's context package, and the JSON of the one the relay last got. */
+    private context;
+    private sentContext;
+    /** The provider's latest handle for taking this call up again after a drop. */
+    private history;
+    private providerState;
+    /** The session said it ended, or failed: a close after that is the end, not a drop. */
+    private sessionOver;
+    /** When the call dropped, for the time away a resume reports. */
+    private droppedAt;
+    /** When the call that could not reconnect dropped: the next call takes it up again. */
+    private lostAt;
+    private reconnectTries;
+    private reconnectTimer;
+    constructor(options: LiveClientOptions);
+    private setStatus;
+    /** Still opening: the first connect, or a try at taking a dropped call up again. */
+    private get opening();
+    private setConnectPhase;
+    private clearConnectTimeout;
+    private detachWebSocket;
+    private clearReconnect;
+    private teardownConnection;
+    /** End the session on `error`: a failed connect or a lost socket. */
+    private failSession;
+    private createSocket;
+    /**
+     * Start a call. After one that dropped and could not reconnect, it takes that call up again,
+     * so the agent neither greets twice nor forgets; after a hang-up it is a new call.
+     */
+    connect(): Promise<void>;
+    /** A lost call that cannot be taken up again, its handle refused or expired, starts as a new one. */
+    private startOver;
+    /** The first message: the call's values, and the handle and time away when it takes a dropped call up again. */
+    private openMessage;
+    /** The socket closed while the call was open: take it up again if the provider gave a handle, else end. */
+    private handleDrop;
+    private scheduleReconnect;
+    /** A try at opening failed: a reconnect waits and tries again, a first connect fails the call. */
+    private failOpen;
+    private open;
+    private ensureAudioContext;
+    private activateMicrophone;
+    private sendMicFrame;
+    private reportMicVolume;
+    private forwardMicBuffer;
+    private setupMicrophonePipeline;
+    private ensurePlaybackBus;
+    private stopPlaybackMeter;
+    private startPlaybackMeter;
+    private processServerEnvelope;
+    private enqueueServerMessage;
+    private handleReadyEnvelope;
+    /** The call is open, first or again: the page's context catches up with what changed meanwhile. */
+    private becomeReady;
+    private handleExecuteToolResultEnvelope;
+    private tryHandleControlEnvelope;
+    private collectInboundTurn;
+    private processTurnEvent;
+    private handleEvidenceTurnEvent;
+    private handleSessionTurnEvent;
+    /** Hand the session's closing and end to the host; `true` when it was one of them. */
+    private notifySessionLifecycle;
+    /** Status to settle into once model speech stops. */
+    private restingStatus;
+    private collectMediaTurnEvent;
+    private handleDoneTurnEvent;
+    private scheduleMediaChunks;
+    /** The relay runs `LiveSession.executeTool`; see `ExecuteToolOnRelay`. */
+    executeToolOnRelay(args: Parameters<ExecuteToolOnRelay>[0]): Promise<LiveToolStep>;
+    private enqueueAudioChunk;
+    cancelPlayback(): void;
+    toggleMute(): boolean;
+    sendText(text: string): void;
+    /**
+     * The call's slot values. An open call is taken up again with them when the provider gave a
+     * handle; with none, the next call has them.
+     */
+    setSlots(slots: Record<string, string> | undefined): void;
+    /**
+     * Replace the page's context package. In a call it reaches the agent as
+     * background, with no reply; before one, it goes with the call's opening.
+     */
+    setContext(context: unknown): void;
+    private flushContext;
+    sendVideo(data: string, mimeType?: string): void;
+    disconnect(): void;
+    private cleanupAudio;
+}