@tanstack/ai-react 0.6.4 → 0.7.1

package/dist/esm/index.d.ts

@@ -1,5 +1,7 @@
 export { useChat } from './use-chat.js';
+export { useRealtimeChat } from './use-realtime-chat.js';
 export type { UseChatOptions, UseChatReturn, UIMessage, ChatRequestBody, } from './types.js';
+export type { UseRealtimeChatOptions, UseRealtimeChatReturn, } from './realtime-types.js';
 export { useGeneration } from './use-generation.js';
 export type { UseGenerationOptions, UseGenerationReturn, } from './use-generation.js';
 export { useGenerateImage } from './use-generate-image.js';

package/dist/esm/index.js

@@ -1,4 +1,5 @@
 import { useChat } from "./use-chat.js";
+import { useRealtimeChat } from "./use-realtime-chat.js";
 import { useGeneration } from "./use-generation.js";
 import { useGenerateImage } from "./use-generate-image.js";
 import { useGenerateSpeech } from "./use-generate-speech.js";
@@ -16,6 +17,7 @@ export {
   useGenerateSpeech,
   useGenerateVideo,
   useGeneration,
+  useRealtimeChat,
   useSummarize,
   useTranscription
 };

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;"}
+{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;"}

package/dist/esm/realtime-types.d.ts

@@ -0,0 +1,109 @@
+import { AnyClientTool, RealtimeMessage, RealtimeMode, RealtimeStatus, RealtimeToken } from '@tanstack/ai';
+import { RealtimeAdapter } from '@tanstack/ai-client';
+/**
+ * Options for the useRealtimeChat hook.
+ */
+export interface UseRealtimeChatOptions {
+    /**
+     * Function to fetch a realtime token from the server.
+     * Called on connect and when token needs refresh.
+     */
+    getToken: () => Promise<RealtimeToken>;
+    /**
+     * The realtime adapter to use (e.g., openaiRealtime())
+     */
+    adapter: RealtimeAdapter;
+    /**
+     * Client-side tools with execution logic
+     */
+    tools?: ReadonlyArray<AnyClientTool>;
+    /**
+     * Auto-play assistant audio (default: true)
+     */
+    autoPlayback?: boolean;
+    /**
+     * Request microphone access on connect (default: true)
+     */
+    autoCapture?: boolean;
+    /**
+     * System instructions for the assistant
+     */
+    instructions?: string;
+    /**
+     * Voice to use for audio output
+     */
+    voice?: string;
+    /**
+     * Voice activity detection mode (default: 'server')
+     */
+    vadMode?: 'server' | 'semantic' | 'manual';
+    /**
+     * Output modalities for responses (e.g., ['audio', 'text'])
+     */
+    outputModalities?: Array<'audio' | 'text'>;
+    /**
+     * Temperature for generation (provider-specific range)
+     */
+    temperature?: number;
+    /**
+     * Maximum number of tokens in a response
+     */
+    maxOutputTokens?: number | 'inf';
+    /**
+     * Eagerness level for semantic VAD ('low', 'medium', 'high')
+     */
+    semanticEagerness?: 'low' | 'medium' | 'high';
+    onConnect?: () => void;
+    onDisconnect?: () => void;
+    onError?: (error: Error) => void;
+    onMessage?: (message: RealtimeMessage) => void;
+    onModeChange?: (mode: RealtimeMode) => void;
+    onInterrupted?: () => void;
+}
+/**
+ * Return type for the useRealtimeChat hook.
+ */
+export interface UseRealtimeChatReturn {
+    /** Current connection status */
+    status: RealtimeStatus;
+    /** Current error, if any */
+    error: Error | null;
+    /** Connect to the realtime session */
+    connect: () => Promise<void>;
+    /** Disconnect from the realtime session */
+    disconnect: () => Promise<void>;
+    /** Current mode (idle, listening, thinking, speaking) */
+    mode: RealtimeMode;
+    /** Conversation messages */
+    messages: Array<RealtimeMessage>;
+    /** User transcript while speaking (before finalized) */
+    pendingUserTranscript: string | null;
+    /** Assistant transcript while speaking (before finalized) */
+    pendingAssistantTranscript: string | null;
+    /** Start listening for voice input (manual VAD mode) */
+    startListening: () => void;
+    /** Stop listening for voice input (manual VAD mode) */
+    stopListening: () => void;
+    /** Interrupt the current assistant response */
+    interrupt: () => void;
+    /** Send a text message instead of voice */
+    sendText: (text: string) => void;
+    /** Send an image to the conversation */
+    sendImage: (imageData: string, mimeType: string) => void;
+    /** Current input (microphone) volume level */
+    inputLevel: number;
+    /** Current output (speaker) volume level */
+    outputLevel: number;
+    /** Get frequency data for input audio visualization */
+    getInputFrequencyData: () => Uint8Array;
+    /** Get frequency data for output audio visualization */
+    getOutputFrequencyData: () => Uint8Array;
+    /** Get time domain data for input waveform */
+    getInputTimeDomainData: () => Uint8Array;
+    /** Get time domain data for output waveform */
+    getOutputTimeDomainData: () => Uint8Array;
+    /** Current VAD mode */
+    vadMode: 'server' | 'semantic' | 'manual';
+    /** Change VAD mode at runtime */
+    setVADMode: (mode: 'server' | 'semantic' | 'manual') => void;
+}

package/dist/esm/use-realtime-chat.d.ts

@@ -0,0 +1,42 @@
+import { UseRealtimeChatOptions, UseRealtimeChatReturn } from './realtime-types.js';
+/**
+ * React hook for realtime voice conversations.
+ *
+ * Provides a simple interface for voice-to-voice AI interactions
+ * with support for multiple providers (OpenAI, ElevenLabs, etc.).
+ *
+ * @param options - Configuration options including adapter and callbacks
+ * @returns Hook return value with state and control methods
+ *
+ * @example
+ * ```typescript
+ * import { useRealtimeChat } from '@tanstack/ai-react'
+ * import { openaiRealtime } from '@tanstack/ai-openai'
+ *
+ * function VoiceChat() {
+ *   const {
+ *     status,
+ *     mode,
+ *     messages,
+ *     connect,
+ *     disconnect,
+ *     inputLevel,
+ *     outputLevel,
+ *   } = useRealtimeChat({
+ *     getToken: () => fetch('/api/realtime-token').then(r => r.json()),
+ *     adapter: openaiRealtime(),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <p>Status: {status}</p>
+ *       <p>Mode: {mode}</p>
+ *       <button onClick={status === 'idle' ? connect : disconnect}>
+ *         {status === 'idle' ? 'Start' : 'Stop'}
+ *       </button>
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export declare function useRealtimeChat(options: UseRealtimeChatOptions): UseRealtimeChatReturn;

package/dist/esm/use-realtime-chat.js

@@ -0,0 +1,174 @@
+import { useState, useRef, useEffect, useCallback } from "react";
+import { RealtimeClient } from "@tanstack/ai-client";
+const emptyFrequencyData = new Uint8Array(128);
+const emptyTimeDomainData = new Uint8Array(128).fill(128);
+function useRealtimeChat(options) {
+  const [status, setStatus] = useState("idle");
+  const [mode, setMode] = useState("idle");
+  const [messages, setMessages] = useState([]);
+  const [pendingUserTranscript, setPendingUserTranscript] = useState(null);
+  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState(null);
+  const [error, setError] = useState(null);
+  const [inputLevel, setInputLevel] = useState(0);
+  const [outputLevel, setOutputLevel] = useState(0);
+  const [vadMode, setVADModeState] = useState(
+    options.vadMode ?? "server"
+  );
+  const clientRef = useRef(null);
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const animationFrameRef = useRef(null);
+  if (!clientRef.current) {
+    clientRef.current = new RealtimeClient({
+      getToken: optionsRef.current.getToken,
+      adapter: optionsRef.current.adapter,
+      tools: optionsRef.current.tools,
+      instructions: optionsRef.current.instructions,
+      voice: optionsRef.current.voice,
+      autoPlayback: optionsRef.current.autoPlayback,
+      autoCapture: optionsRef.current.autoCapture,
+      vadMode: optionsRef.current.vadMode,
+      outputModalities: optionsRef.current.outputModalities,
+      temperature: optionsRef.current.temperature,
+      maxOutputTokens: optionsRef.current.maxOutputTokens,
+      semanticEagerness: optionsRef.current.semanticEagerness,
+      onStatusChange: (newStatus) => {
+        setStatus(newStatus);
+      },
+      onModeChange: (newMode) => {
+        setMode(newMode);
+        optionsRef.current.onModeChange?.(newMode);
+      },
+      onMessage: (message) => {
+        setMessages((prev) => [...prev, message]);
+        optionsRef.current.onMessage?.(message);
+      },
+      onError: (err) => {
+        setError(err);
+        optionsRef.current.onError?.(err);
+      },
+      onConnect: () => {
+        setError(null);
+        optionsRef.current.onConnect?.();
+      },
+      onDisconnect: () => {
+        optionsRef.current.onDisconnect?.();
+      },
+      onInterrupted: () => {
+        setPendingAssistantTranscript(null);
+        optionsRef.current.onInterrupted?.();
+      }
+    });
+    clientRef.current.onStateChange((state) => {
+      setPendingUserTranscript(state.pendingUserTranscript);
+      setPendingAssistantTranscript(state.pendingAssistantTranscript);
+    });
+  }
+  const client = clientRef.current;
+  useEffect(() => {
+    function updateLevels() {
+      if (clientRef.current?.audio) {
+        setInputLevel(clientRef.current.audio.inputLevel);
+        setOutputLevel(clientRef.current.audio.outputLevel);
+      }
+      animationFrameRef.current = requestAnimationFrame(updateLevels);
+    }
+    if (status === "connected") {
+      updateLevels();
+    }
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current);
+        animationFrameRef.current = null;
+      }
+    };
+  }, [status]);
+  useEffect(() => {
+    return () => {
+      clientRef.current?.destroy();
+    };
+  }, []);
+  const connect = useCallback(async () => {
+    setError(null);
+    setMessages([]);
+    setPendingUserTranscript(null);
+    setPendingAssistantTranscript(null);
+    await client.connect();
+  }, [client]);
+  const disconnect = useCallback(async () => {
+    await client.disconnect();
+  }, [client]);
+  const startListening = useCallback(() => {
+    client.startListening();
+  }, [client]);
+  const stopListening = useCallback(() => {
+    client.stopListening();
+  }, [client]);
+  const interrupt = useCallback(() => {
+    client.interrupt();
+  }, [client]);
+  const sendText = useCallback(
+    (text) => {
+      client.sendText(text);
+    },
+    [client]
+  );
+  const sendImage = useCallback(
+    (imageData, mimeType) => {
+      client.sendImage(imageData, mimeType);
+    },
+    [client]
+  );
+  const getInputFrequencyData = useCallback(() => {
+    return clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData;
+  }, []);
+  const getOutputFrequencyData = useCallback(() => {
+    return clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData;
+  }, []);
+  const getInputTimeDomainData = useCallback(() => {
+    return clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData;
+  }, []);
+  const getOutputTimeDomainData = useCallback(() => {
+    return clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData;
+  }, []);
+  const setVADMode = useCallback(
+    (newMode) => {
+      setVADModeState(newMode);
+    },
+    []
+  );
+  return {
+    // Connection state
+    status,
+    error,
+    connect,
+    disconnect,
+    // Conversation state
+    mode,
+    messages,
+    pendingUserTranscript,
+    pendingAssistantTranscript,
+    // Voice control
+    startListening,
+    stopListening,
+    interrupt,
+    // Text input
+    sendText,
+    // Image input
+    sendImage,
+    // Audio visualization
+    inputLevel,
+    outputLevel,
+    getInputFrequencyData,
+    getOutputFrequencyData,
+    getInputTimeDomainData,
+    getOutputTimeDomainData,
+    // VAD control
+    vadMode,
+    setVADMode
+  };
+}
+export {
+  useRealtimeChat
+};
+//# sourceMappingURL=use-realtime-chat.js.map

package/dist/esm/use-realtime-chat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-realtime-chat.js","sources":["../../src/use-realtime-chat.ts"],"sourcesContent":["import { useCallback, useEffect, useRef, useState } from 'react'\nimport { RealtimeClient } from '@tanstack/ai-client'\nimport type {\n  RealtimeMessage,\n  RealtimeMode,\n  RealtimeStatus,\n} from '@tanstack/ai'\nimport type {\n  UseRealtimeChatOptions,\n  UseRealtimeChatReturn,\n} from './realtime-types'\n\n// Empty frequency data for when client is not connected\nconst emptyFrequencyData = new Uint8Array(128)\nconst emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n/**\n * React hook for realtime voice conversations.\n *\n * Provides a simple interface for voice-to-voice AI interactions\n * with support for multiple providers (OpenAI, ElevenLabs, etc.).\n *\n * @param options - Configuration options including adapter and callbacks\n * @returns Hook return value with state and control methods\n *\n * @example\n * ```typescript\n * import { useRealtimeChat } from '@tanstack/ai-react'\n * import { openaiRealtime } from '@tanstack/ai-openai'\n *\n * function VoiceChat() {\n *   const {\n *     status,\n *     mode,\n *     messages,\n *     connect,\n *     disconnect,\n *     inputLevel,\n *     outputLevel,\n *   } = useRealtimeChat({\n *     getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n *     adapter: openaiRealtime(),\n *   })\n *\n *   return (\n *     <div>\n *       <p>Status: {status}</p>\n *       <p>Mode: {mode}</p>\n *       <button onClick={status === 'idle' ? connect : disconnect}>\n *         {status === 'idle' ? 'Start' : 'Stop'}\n *       </button>\n *     </div>\n *   )\n * }\n * ```\n */\nexport function useRealtimeChat(\n  options: UseRealtimeChatOptions,\n): UseRealtimeChatReturn {\n  // State\n  const [status, setStatus] = useState<RealtimeStatus>('idle')\n  const [mode, setMode] = useState<RealtimeMode>('idle')\n  const [messages, setMessages] = useState<Array<RealtimeMessage>>([])\n  const [pendingUserTranscript, setPendingUserTranscript] = useState<\n    string | null\n  >(null)\n  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState<\n    string | null\n  >(null)\n  const [error, setError] = useState<Error | null>(null)\n  const [inputLevel, setInputLevel] = useState(0)\n  const [outputLevel, setOutputLevel] = useState(0)\n  const [vadMode, setVADModeState] = useState<'server' | 'semantic' | 'manual'>(\n    options.vadMode ?? 'server',\n  )\n\n  // Refs\n  const clientRef = useRef<RealtimeClient | null>(null)\n  const optionsRef = useRef(options)\n  optionsRef.current = options\n  const animationFrameRef = useRef<number | null>(null)\n\n  // Create client instance - use ref to ensure we reuse the same instance\n  // This handles React StrictMode double-rendering\n  if (!clientRef.current) {\n    clientRef.current = new RealtimeClient({\n      getToken: optionsRef.current.getToken,\n      adapter: optionsRef.current.adapter,\n      tools: optionsRef.current.tools,\n      instructions: optionsRef.current.instructions,\n      voice: optionsRef.current.voice,\n      autoPlayback: optionsRef.current.autoPlayback,\n      autoCapture: optionsRef.current.autoCapture,\n      vadMode: optionsRef.current.vadMode,\n      outputModalities: optionsRef.current.outputModalities,\n      temperature: optionsRef.current.temperature,\n      maxOutputTokens: optionsRef.current.maxOutputTokens,\n      semanticEagerness: optionsRef.current.semanticEagerness,\n      onStatusChange: (newStatus) => {\n        setStatus(newStatus)\n      },\n      onModeChange: (newMode) => {\n        setMode(newMode)\n        optionsRef.current.onModeChange?.(newMode)\n      },\n      onMessage: (message) => {\n        setMessages((prev) => [...prev, message])\n        optionsRef.current.onMessage?.(message)\n      },\n      onError: (err) => {\n        setError(err)\n        optionsRef.current.onError?.(err)\n      },\n      onConnect: () => {\n        setError(null)\n        optionsRef.current.onConnect?.()\n      },\n      onDisconnect: () => {\n        optionsRef.current.onDisconnect?.()\n      },\n      onInterrupted: () => {\n        setPendingAssistantTranscript(null)\n        optionsRef.current.onInterrupted?.()\n      },\n    })\n\n    // Subscribe to state changes for transcripts\n    clientRef.current.onStateChange((state) => {\n      setPendingUserTranscript(state.pendingUserTranscript)\n      setPendingAssistantTranscript(state.pendingAssistantTranscript)\n    })\n  }\n\n  const client = clientRef.current\n\n  // Audio level animation loop\n  useEffect(() => {\n    function updateLevels() {\n      if (clientRef.current?.audio) {\n        setInputLevel(clientRef.current.audio.inputLevel)\n        setOutputLevel(clientRef.current.audio.outputLevel)\n      }\n      animationFrameRef.current = requestAnimationFrame(updateLevels)\n    }\n\n    if (status === 'connected') {\n      updateLevels()\n    }\n\n    return () => {\n      if (animationFrameRef.current) {\n        cancelAnimationFrame(animationFrameRef.current)\n        animationFrameRef.current = null\n      }\n    }\n  }, [status])\n\n  // Cleanup on unmount\n  useEffect(() => {\n    return () => {\n      clientRef.current?.destroy()\n    }\n  }, [])\n\n  // Connection methods\n  const connect = useCallback(async () => {\n    setError(null)\n    setMessages([])\n    setPendingUserTranscript(null)\n    setPendingAssistantTranscript(null)\n    await client.connect()\n  }, [client])\n\n  const disconnect = useCallback(async () => {\n    await client.disconnect()\n  }, [client])\n\n  // Voice control methods\n  const startListening = useCallback(() => {\n    client.startListening()\n  }, [client])\n\n  const stopListening = useCallback(() => {\n    client.stopListening()\n  }, [client])\n\n  const interrupt = useCallback(() => {\n    client.interrupt()\n  }, [client])\n\n  // Text input\n  const sendText = useCallback(\n    (text: string) => {\n      client.sendText(text)\n    },\n    [client],\n  )\n\n  // Image input\n  const sendImage = useCallback(\n    (imageData: string, mimeType: string) => {\n      client.sendImage(imageData, mimeType)\n    },\n    [client],\n  )\n\n  // Audio visualization\n  const getInputFrequencyData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData\n    )\n  }, [])\n\n  const getOutputFrequencyData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData\n    )\n  }, [])\n\n  const getInputTimeDomainData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData\n    )\n  }, [])\n\n  const getOutputTimeDomainData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData\n    )\n  }, [])\n\n  // VAD mode control\n  const setVADMode = useCallback(\n    (newMode: 'server' | 'semantic' | 'manual') => {\n      setVADModeState(newMode)\n      // TODO: Update session config if connected\n    },\n    [],\n  )\n\n  return {\n    // Connection state\n    status,\n    error,\n    connect,\n    disconnect,\n\n    // Conversation state\n    mode,\n    messages,\n    pendingUserTranscript,\n    pendingAssistantTranscript,\n\n    // Voice control\n    startListening,\n    stopListening,\n    interrupt,\n\n    // Text input\n    sendText,\n\n    // Image input\n    sendImage,\n\n    // Audio visualization\n    inputLevel,\n    outputLevel,\n    getInputFrequencyData,\n    getOutputFrequencyData,\n    getInputTimeDomainData,\n    getOutputTimeDomainData,\n\n    // VAD control\n    vadMode,\n    setVADMode,\n  }\n}\n"],"names":[],"mappings":";;AAaA,MAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,MAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AA0CjD,SAAS,gBACd,SACuB;AAEvB,QAAM,CAAC,QAAQ,SAAS,IAAI,SAAyB,MAAM;AAC3D,QAAM,CAAC,MAAM,OAAO,IAAI,SAAuB,MAAM;AACrD,QAAM,CAAC,UAAU,WAAW,IAAI,SAAiC,CAAA,CAAE;AACnE,QAAM,CAAC,uBAAuB,wBAAwB,IAAI,SAExD,IAAI;AACN,QAAM,CAAC,4BAA4B,6BAA6B,IAAI,SAElE,IAAI;AACN,QAAM,CAAC,OAAO,QAAQ,IAAI,SAAuB,IAAI;AACrD,QAAM,CAAC,YAAY,aAAa,IAAI,SAAS,CAAC;AAC9C,QAAM,CAAC,aAAa,cAAc,IAAI,SAAS,CAAC;AAChD,QAAM,CAAC,SAAS,eAAe,IAAI;AAAA,IACjC,QAAQ,WAAW;AAAA,EAAA;AAIrB,QAAM,YAAY,OAA8B,IAAI;AACpD,QAAM,aAAa,OAAO,OAAO;AACjC,aAAW,UAAU;AACrB,QAAM,oBAAoB,OAAsB,IAAI;AAIpD,MAAI,CAAC,UAAU,SAAS;AACtB,cAAU,UAAU,IAAI,eAAe;AAAA,MACrC,UAAU,WAAW,QAAQ;AAAA,MAC7B,SAAS,WAAW,QAAQ;AAAA,MAC5B,OAAO,WAAW,QAAQ;AAAA,MAC1B,cAAc,WAAW,QAAQ;AAAA,MACjC,OAAO,WAAW,QAAQ;AAAA,MAC1B,cAAc,WAAW,QAAQ;AAAA,MACjC,aAAa,WAAW,QAAQ;AAAA,MAChC,SAAS,WAAW,QAAQ;AAAA,MAC5B,kBAAkB,WAAW,QAAQ;AAAA,MACrC,aAAa,WAAW,QAAQ;AAAA,MAChC,iBAAiB,WAAW,QAAQ;AAAA,MACpC,mBAAmB,WAAW,QAAQ;AAAA,MACtC,gBAAgB,CAAC,cAAc;AAC7B,kBAAU,SAAS;AAAA,MACrB;AAAA,MACA,cAAc,CAAC,YAAY;AACzB,gBAAQ,OAAO;AACf,mBAAW,QAAQ,eAAe,OAAO;AAAA,MAC3C;AAAA,MACA,WAAW,CAAC,YAAY;AACtB,oBAAY,CAAC,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC;AACxC,mBAAW,QAAQ,YAAY,OAAO;AAAA,MACxC;AAAA,MACA,SAAS,CAAC,QAAQ;AAChB,iBAAS,GAAG;AACZ,mBAAW,QAAQ,UAAU,GAAG;AAAA,MAClC;AAAA,MACA,WAAW,MAAM;AACf,iBAAS,IAAI;AACb,mBAAW,QAAQ,YAAA;AAAA,MACrB;AAAA,MACA,cAAc,MAAM;AAClB,mBAAW,QAAQ,eAAA;AAAA,MACrB;AAAA,MACA,eAAe,MAAM;AACnB,sCAA8B,IAAI;AAClC,mBAAW,QAAQ,gBAAA;AAAA,MACrB;AAAA,IAAA,CACD;AAGD,cAAU,QAAQ,cAAc,CAAC,UAAU;AACzC,+BAAyB,MAAM,qBAAqB;AACpD,oCAA8B,MAAM,0BAA0B;AAAA,IAChE,CAAC;AAAA,EACH;AAEA,QAAM,SAAS,UAAU;AAGzB,YAAU,MAAM;AACd,aAAS,eAAe;AACtB,UAAI,UAAU,SAAS,OAAO;AAC5B,sBAAc,UAAU,QAAQ,MAAM,UAAU;AAChD,uBAAe,UAAU,QAAQ,MAAM,WAAW;AAAA,MACpD;AACA,wBAAkB,UAAU,sBAAsB,YAAY;AAAA,IAChE;AAEA,QAAI,WAAW,aAAa;AAC1B,mBAAA;AAAA,IACF;AAEA,WAAO,MAAM;AACX,UAAI,kBAAkB,SAAS;AAC7B,6BAAqB,kBAAkB,OAAO;AAC9C,0BAAkB,UAAU;AAAA,MAC9B;AAAA,IACF;AAAA,EACF,GAAG,CAAC,MAAM,CAAC;AAGX,YAAU,MAAM;AACd,WAAO,MAAM;AACX,gBAAU,SAAS,QAAA;AAAA,IACrB;AAAA,EACF,GAAG,CAAA,CAAE;AAGL,QAAM,UAAU,YAAY,YAAY;AACtC,aAAS,IAAI;AACb,gBAAY,CAAA,CAAE;AACd,6BAAyB,IAAI;AAC7B,kCAA8B,IAAI;AAClC,UAAM,OAAO,QAAA;AAAA,EACf,GAAG,CAAC,MAAM,CAAC;AAEX,QAAM,aAAa,YAAY,YAAY;AACzC,UAAM,OAAO,WAAA;AAAA,EACf,GAAG,CAAC,MAAM,CAAC;AAGX,QAAM,iBAAiB,YAAY,MAAM;AACvC,WAAO,eAAA;AAAA,EACT,GAAG,CAAC,MAAM,CAAC;AAEX,QAAM,gBAAgB,YAAY,MAAM;AACtC,WAAO,cAAA;AAAA,EACT,GAAG,CAAC,MAAM,CAAC;AAEX,QAAM,YAAY,YAAY,MAAM;AAClC,WAAO,UAAA;AAAA,EACT,GAAG,CAAC,MAAM,CAAC;AAGX,QAAM,WAAW;AAAA,IACf,CAAC,SAAiB;AAChB,aAAO,SAAS,IAAI;AAAA,IACtB;AAAA,IACA,CAAC,MAAM;AAAA,EAAA;AAIT,QAAM,YAAY;AAAA,IAChB,CAAC,WAAmB,aAAqB;AACvC,aAAO,UAAU,WAAW,QAAQ;AAAA,IACtC;AAAA,IACA,CAAC,MAAM;AAAA,EAAA;AAIT,QAAM,wBAAwB,YAAY,MAAM;AAC9C,WACE,UAAU,SAAS,OAAO,sBAAA,KAA2B;AAAA,EAEzD,GAAG,CAAA,CAAE;AAEL,QAAM,yBAAyB,YAAY,MAAM;AAC/C,WACE,UAAU,SAAS,OAAO,uBAAA,KAA4B;AAAA,EAE1D,GAAG,CAAA,CAAE;AAEL,QAAM,yBAAyB,YAAY,MAAM;AAC/C,WACE,UAAU,SAAS,OAAO,uBAAA,KAA4B;AAAA,EAE1D,GAAG,CAAA,CAAE;AAEL,QAAM,0BAA0B,YAAY,MAAM;AAChD,WACE,UAAU,SAAS,OAAO,wBAAA,KAA6B;AAAA,EAE3D,GAAG,CAAA,CAAE;AAGL,QAAM,aAAa;AAAA,IACjB,CAAC,YAA8C;AAC7C,sBAAgB,OAAO;AAAA,IAEzB;AAAA,IACA,CAAA;AAAA,EAAC;AAGH,SAAO;AAAA;AAAA,IAEL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA;AAAA,IAGA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai-react",
-  "version": "0.6.4",
+  "version": "0.7.1",
   "description": "React hooks for TanStack AI",
   "author": "",
   "license": "MIT",
@@ -31,12 +31,12 @@
     "streaming"
   ],
   "dependencies": {
-    "@tanstack/ai-client": "0.6.0"
+    "@tanstack/ai-client": "0.7.1"
   },
   "peerDependencies": {
     "@types/react": ">=18.0.0",
     "react": ">=18.0.0",
-    "@tanstack/ai": "^0.6.3"
+    "@tanstack/ai": "^0.8.0"
   },
   "devDependencies": {
     "@testing-library/react": "^16.3.0",
@@ -45,7 +45,7 @@
     "jsdom": "^27.2.0",
     "react": "^19.2.3",
     "vite": "^7.2.7",
-    "@tanstack/ai": "0.6.3"
+    "@tanstack/ai": "0.8.0"
   },
   "scripts": {
     "clean": "premove ./build ./dist",

package/src/index.ts

@@ -1,10 +1,15 @@
 export { useChat } from './use-chat'
+export { useRealtimeChat } from './use-realtime-chat'
 export type {
   UseChatOptions,
   UseChatReturn,
   UIMessage,
   ChatRequestBody,
 } from './types'
+export type {
+  UseRealtimeChatOptions,
+  UseRealtimeChatReturn,
+} from './realtime-types'
 
 // Generation hooks
 export { useGeneration } from './use-generation'

package/src/realtime-types.ts

@@ -0,0 +1,143 @@
+import type {
+  AnyClientTool,
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeStatus,
+  RealtimeToken,
+} from '@tanstack/ai'
+import type { RealtimeAdapter } from '@tanstack/ai-client'
+
+/**
+ * Options for the useRealtimeChat hook.
+ */
+export interface UseRealtimeChatOptions {
+  /**
+   * Function to fetch a realtime token from the server.
+   * Called on connect and when token needs refresh.
+   */
+  getToken: () => Promise<RealtimeToken>
+
+  /**
+   * The realtime adapter to use (e.g., openaiRealtime())
+   */
+  adapter: RealtimeAdapter
+
+  /**
+   * Client-side tools with execution logic
+   */
+  tools?: ReadonlyArray<AnyClientTool>
+
+  /**
+   * Auto-play assistant audio (default: true)
+   */
+  autoPlayback?: boolean
+
+  /**
+   * Request microphone access on connect (default: true)
+   */
+  autoCapture?: boolean
+
+  /**
+   * System instructions for the assistant
+   */
+  instructions?: string
+
+  /**
+   * Voice to use for audio output
+   */
+  voice?: string
+
+  /**
+   * Voice activity detection mode (default: 'server')
+   */
+  vadMode?: 'server' | 'semantic' | 'manual'
+
+  /**
+   * Output modalities for responses (e.g., ['audio', 'text'])
+   */
+  outputModalities?: Array<'audio' | 'text'>
+
+  /**
+   * Temperature for generation (provider-specific range)
+   */
+  temperature?: number
+
+  /**
+   * Maximum number of tokens in a response
+   */
+  maxOutputTokens?: number | 'inf'
+
+  /**
+   * Eagerness level for semantic VAD ('low', 'medium', 'high')
+   */
+  semanticEagerness?: 'low' | 'medium' | 'high'
+
+  // Callbacks
+  onConnect?: () => void
+  onDisconnect?: () => void
+  onError?: (error: Error) => void
+  onMessage?: (message: RealtimeMessage) => void
+  onModeChange?: (mode: RealtimeMode) => void
+  onInterrupted?: () => void
+}
+
+/**
+ * Return type for the useRealtimeChat hook.
+ */
+export interface UseRealtimeChatReturn {
+  // Connection state
+  /** Current connection status */
+  status: RealtimeStatus
+  /** Current error, if any */
+  error: Error | null
+  /** Connect to the realtime session */
+  connect: () => Promise<void>
+  /** Disconnect from the realtime session */
+  disconnect: () => Promise<void>
+
+  // Conversation state
+  /** Current mode (idle, listening, thinking, speaking) */
+  mode: RealtimeMode
+  /** Conversation messages */
+  messages: Array<RealtimeMessage>
+  /** User transcript while speaking (before finalized) */
+  pendingUserTranscript: string | null
+  /** Assistant transcript while speaking (before finalized) */
+  pendingAssistantTranscript: string | null
+
+  // Voice control
+  /** Start listening for voice input (manual VAD mode) */
+  startListening: () => void
+  /** Stop listening for voice input (manual VAD mode) */
+  stopListening: () => void
+  /** Interrupt the current assistant response */
+  interrupt: () => void
+
+  // Text input
+  /** Send a text message instead of voice */
+  sendText: (text: string) => void
+
+  // Image input
+  /** Send an image to the conversation */
+  sendImage: (imageData: string, mimeType: string) => void
+
+  // Audio visualization (0-1 normalized)
+  /** Current input (microphone) volume level */
+  inputLevel: number
+  /** Current output (speaker) volume level */
+  outputLevel: number
+  /** Get frequency data for input audio visualization */
+  getInputFrequencyData: () => Uint8Array
+  /** Get frequency data for output audio visualization */
+  getOutputFrequencyData: () => Uint8Array
+  /** Get time domain data for input waveform */
+  getInputTimeDomainData: () => Uint8Array
+  /** Get time domain data for output waveform */
+  getOutputTimeDomainData: () => Uint8Array
+
+  // VAD control
+  /** Current VAD mode */
+  vadMode: 'server' | 'semantic' | 'manual'
+  /** Change VAD mode at runtime */
+  setVADMode: (mode: 'server' | 'semantic' | 'manual') => void
+}

package/src/use-realtime-chat.ts

@@ -0,0 +1,277 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { RealtimeClient } from '@tanstack/ai-client'
+import type {
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeStatus,
+} from '@tanstack/ai'
+import type {
+  UseRealtimeChatOptions,
+  UseRealtimeChatReturn,
+} from './realtime-types'
+
+// Empty frequency data for when client is not connected
+const emptyFrequencyData = new Uint8Array(128)
+const emptyTimeDomainData = new Uint8Array(128).fill(128)
+
+/**
+ * React hook for realtime voice conversations.
+ *
+ * Provides a simple interface for voice-to-voice AI interactions
+ * with support for multiple providers (OpenAI, ElevenLabs, etc.).
+ *
+ * @param options - Configuration options including adapter and callbacks
+ * @returns Hook return value with state and control methods
+ *
+ * @example
+ * ```typescript
+ * import { useRealtimeChat } from '@tanstack/ai-react'
+ * import { openaiRealtime } from '@tanstack/ai-openai'
+ *
+ * function VoiceChat() {
+ *   const {
+ *     status,
+ *     mode,
+ *     messages,
+ *     connect,
+ *     disconnect,
+ *     inputLevel,
+ *     outputLevel,
+ *   } = useRealtimeChat({
+ *     getToken: () => fetch('/api/realtime-token').then(r => r.json()),
+ *     adapter: openaiRealtime(),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <p>Status: {status}</p>
+ *       <p>Mode: {mode}</p>
+ *       <button onClick={status === 'idle' ? connect : disconnect}>
+ *         {status === 'idle' ? 'Start' : 'Stop'}
+ *       </button>
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useRealtimeChat(
+  options: UseRealtimeChatOptions,
+): UseRealtimeChatReturn {
+  // State
+  const [status, setStatus] = useState<RealtimeStatus>('idle')
+  const [mode, setMode] = useState<RealtimeMode>('idle')
+  const [messages, setMessages] = useState<Array<RealtimeMessage>>([])
+  const [pendingUserTranscript, setPendingUserTranscript] = useState<
+    string | null
+  >(null)
+  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState<
+    string | null
+  >(null)
+  const [error, setError] = useState<Error | null>(null)
+  const [inputLevel, setInputLevel] = useState(0)
+  const [outputLevel, setOutputLevel] = useState(0)
+  const [vadMode, setVADModeState] = useState<'server' | 'semantic' | 'manual'>(
+    options.vadMode ?? 'server',
+  )
+
+  // Refs
+  const clientRef = useRef<RealtimeClient | null>(null)
+  const optionsRef = useRef(options)
+  optionsRef.current = options
+  const animationFrameRef = useRef<number | null>(null)
+
+  // Create client instance - use ref to ensure we reuse the same instance
+  // This handles React StrictMode double-rendering
+  if (!clientRef.current) {
+    clientRef.current = new RealtimeClient({
+      getToken: optionsRef.current.getToken,
+      adapter: optionsRef.current.adapter,
+      tools: optionsRef.current.tools,
+      instructions: optionsRef.current.instructions,
+      voice: optionsRef.current.voice,
+      autoPlayback: optionsRef.current.autoPlayback,
+      autoCapture: optionsRef.current.autoCapture,
+      vadMode: optionsRef.current.vadMode,
+      outputModalities: optionsRef.current.outputModalities,
+      temperature: optionsRef.current.temperature,
+      maxOutputTokens: optionsRef.current.maxOutputTokens,
+      semanticEagerness: optionsRef.current.semanticEagerness,
+      onStatusChange: (newStatus) => {
+        setStatus(newStatus)
+      },
+      onModeChange: (newMode) => {
+        setMode(newMode)
+        optionsRef.current.onModeChange?.(newMode)
+      },
+      onMessage: (message) => {
+        setMessages((prev) => [...prev, message])
+        optionsRef.current.onMessage?.(message)
+      },
+      onError: (err) => {
+        setError(err)
+        optionsRef.current.onError?.(err)
+      },
+      onConnect: () => {
+        setError(null)
+        optionsRef.current.onConnect?.()
+      },
+      onDisconnect: () => {
+        optionsRef.current.onDisconnect?.()
+      },
+      onInterrupted: () => {
+        setPendingAssistantTranscript(null)
+        optionsRef.current.onInterrupted?.()
+      },
+    })
+
+    // Subscribe to state changes for transcripts
+    clientRef.current.onStateChange((state) => {
+      setPendingUserTranscript(state.pendingUserTranscript)
+      setPendingAssistantTranscript(state.pendingAssistantTranscript)
+    })
+  }
+
+  const client = clientRef.current
+
+  // Audio level animation loop
+  useEffect(() => {
+    function updateLevels() {
+      if (clientRef.current?.audio) {
+        setInputLevel(clientRef.current.audio.inputLevel)
+        setOutputLevel(clientRef.current.audio.outputLevel)
+      }
+      animationFrameRef.current = requestAnimationFrame(updateLevels)
+    }
+
+    if (status === 'connected') {
+      updateLevels()
+    }
+
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current)
+        animationFrameRef.current = null
+      }
+    }
+  }, [status])
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      clientRef.current?.destroy()
+    }
+  }, [])
+
+  // Connection methods
+  const connect = useCallback(async () => {
+    setError(null)
+    setMessages([])
+    setPendingUserTranscript(null)
+    setPendingAssistantTranscript(null)
+    await client.connect()
+  }, [client])
+
+  const disconnect = useCallback(async () => {
+    await client.disconnect()
+  }, [client])
+
+  // Voice control methods
+  const startListening = useCallback(() => {
+    client.startListening()
+  }, [client])
+
+  const stopListening = useCallback(() => {
+    client.stopListening()
+  }, [client])
+
+  const interrupt = useCallback(() => {
+    client.interrupt()
+  }, [client])
+
+  // Text input
+  const sendText = useCallback(
+    (text: string) => {
+      client.sendText(text)
+    },
+    [client],
+  )
+
+  // Image input
+  const sendImage = useCallback(
+    (imageData: string, mimeType: string) => {
+      client.sendImage(imageData, mimeType)
+    },
+    [client],
+  )
+
+  // Audio visualization
+  const getInputFrequencyData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData
+    )
+  }, [])
+
+  const getOutputFrequencyData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData
+    )
+  }, [])
+
+  const getInputTimeDomainData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData
+    )
+  }, [])
+
+  const getOutputTimeDomainData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData
+    )
+  }, [])
+
+  // VAD mode control
+  const setVADMode = useCallback(
+    (newMode: 'server' | 'semantic' | 'manual') => {
+      setVADModeState(newMode)
+      // TODO: Update session config if connected
+    },
+    [],
+  )
+
+  return {
+    // Connection state
+    status,
+    error,
+    connect,
+    disconnect,
+
+    // Conversation state
+    mode,
+    messages,
+    pendingUserTranscript,
+    pendingAssistantTranscript,
+
+    // Voice control
+    startListening,
+    stopListening,
+    interrupt,
+
+    // Text input
+    sendText,
+
+    // Image input
+    sendImage,
+
+    // Audio visualization
+    inputLevel,
+    outputLevel,
+    getInputFrequencyData,
+    getOutputFrequencyData,
+    getInputTimeDomainData,
+    getOutputTimeDomainData,
+
+    // VAD control
+    vadMode,
+    setVADMode,
+  }
+}