@tanstack/ai-react 0.6.4 → 0.7.2

package/src/use-chat.ts

@@ -1,7 +1,7 @@
 import { ChatClient } from '@tanstack/ai-client'
 import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
 import type { AnyClientTool, ModelMessage } from '@tanstack/ai'
-import type { ChatClientState } from '@tanstack/ai-client'
+import type { ChatClientState, ConnectionStatus } from '@tanstack/ai-client'
 
 import type {
   MultimodalContent,
@@ -22,6 +22,10 @@ export function useChat<TTools extends ReadonlyArray<AnyClientTool> = any>(
   const [isLoading, setIsLoading] = useState(false)
   const [error, setError] = useState<Error | undefined>(undefined)
   const [status, setStatus] = useState<ChatClientState>('ready')
+  const [isSubscribed, setIsSubscribed] = useState(false)
+  const [connectionStatus, setConnectionStatus] =
+    useState<ConnectionStatus>('disconnected')
+  const [sessionGenerating, setSessionGenerating] = useState(false)
 
   // Track current messages in a ref to preserve them when client is recreated
   const messagesRef = useRef<Array<UIMessage<TTools>>>(
@@ -78,6 +82,15 @@ export function useChat<TTools extends ReadonlyArray<AnyClientTool> = any>(
       onStatusChange: (status: ChatClientState) => {
         setStatus(status)
       },
+      onSubscriptionChange: (nextIsSubscribed: boolean) => {
+        setIsSubscribed(nextIsSubscribed)
+      },
+      onConnectionStatusChange: (nextStatus: ConnectionStatus) => {
+        setConnectionStatus(nextStatus)
+      },
+      onSessionGeneratingChange: (isGenerating: boolean) => {
+        setSessionGenerating(isGenerating)
+      },
     })
   }, [clientId])
 
@@ -99,16 +112,29 @@ export function useChat<TTools extends ReadonlyArray<AnyClientTool> = any>(
     }
   }, []) // Only run on mount - initialMessages are handled by ChatClient constructor
 
+  // Keep connection lifecycle opt-in and explicit.
+  useEffect(() => {
+    if (options.live) {
+      client.subscribe()
+    } else {
+      client.unsubscribe()
+    }
+  }, [client, options.live])
+
   // Cleanup on unmount: stop any in-flight requests
   // Note: We only cleanup when client changes or component unmounts.
   // DO NOT include isLoading in dependencies - that would cause the cleanup
   // to run when isLoading changes, aborting continuation requests.
   useEffect(() => {
     return () => {
-      // Stop any active generation when component unmounts or client changes
-      client.stop()
+      // live mode owns the connection lifecycle; non-live keeps request-only stop.
+      if (options.live) {
+        client.unsubscribe()
+      } else {
+        client.stop()
+      }
     }
-  }, [client])
+  }, [client, options.live])
 
   // Note: Callback options (onResponse, onChunk, onFinish, onError, onToolCall)
   // are captured at client creation time. Changes to these callbacks require
@@ -176,6 +202,9 @@ export function useChat<TTools extends ReadonlyArray<AnyClientTool> = any>(
     isLoading,
     error,
     status,
+    isSubscribed,
+    connectionStatus,
+    sessionGenerating,
     setMessages: setMessagesManually,
     clear,
     addToolResult,

package/src/use-generate-image.ts

@@ -1,7 +1,7 @@
 import { useGeneration } from './use-generation'
 import type { ImageGenerationResult, StreamChunk } from '@tanstack/ai'
 import type {
-  ConnectionAdapter,
+  ConnectConnectionAdapter,
   GenerationClientState,
   GenerationFetcher,
   ImageGenerateInput,
@@ -14,13 +14,13 @@ import type {
  * @template TOutput - The output type after optional transform (defaults to ImageGenerationResult)
  */
 export interface UseGenerateImageOptions<TOutput = ImageGenerationResult> {
-  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
-  connection?: ConnectionAdapter
+  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectConnectionAdapter
   /** Direct async function for image generation */
   fetcher?: GenerationFetcher<ImageGenerateInput, ImageGenerationResult>
   /** Unique identifier for this generation instance */
   id?: string
-  /** Additional body parameters to send with ConnectionAdapter requests */
+  /** Additional body parameters to send with connect-based adapter requests */
   body?: Record<string, any>
   /**
    * Callback when images are generated. Can optionally return a transformed value.
@@ -34,7 +34,7 @@ export interface UseGenerateImageOptions<TOutput = ImageGenerationResult> {
   onError?: (error: Error) => void
   /** Callback when progress is reported (0-100) */
   onProgress?: (progress: number, message?: string) => void
-  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  /** Callback for each stream chunk (connect-based adapter mode only) */
   onChunk?: (chunk: StreamChunk) => void
 }
 
@@ -64,7 +64,7 @@ export interface UseGenerateImageReturn<TOutput = ImageGenerationResult> {
  * React hook for generating images using AI models.
  *
  * Supports two transport modes:
- * - **ConnectionAdapter** — Streaming transport (SSE, HTTP stream, custom)
+ * - **ConnectConnectionAdapter** — Streaming transport (SSE, HTTP stream, custom)
  * - **Fetcher** — Direct async function call
  *
  * @example

package/src/use-generate-speech.ts

@@ -1,7 +1,7 @@
 import { useGeneration } from './use-generation'
 import type { StreamChunk, TTSResult } from '@tanstack/ai'
 import type {
-  ConnectionAdapter,
+  ConnectConnectionAdapter,
   GenerationClientState,
   GenerationFetcher,
   InferGenerationOutput,
@@ -14,13 +14,13 @@ import type {
  * @template TOutput - The output type after optional transform (defaults to TTSResult)
  */
 export interface UseGenerateSpeechOptions<TOutput = TTSResult> {
-  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
-  connection?: ConnectionAdapter
+  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectConnectionAdapter
   /** Direct async function for speech generation */
   fetcher?: GenerationFetcher<SpeechGenerateInput, TTSResult>
   /** Unique identifier for this generation instance */
   id?: string
-  /** Additional body parameters to send with ConnectionAdapter requests */
+  /** Additional body parameters to send with connect-based adapter requests */
   body?: Record<string, any>
   /**
    * Callback when speech is generated. Can optionally return a transformed value.
@@ -34,7 +34,7 @@ export interface UseGenerateSpeechOptions<TOutput = TTSResult> {
   onError?: (error: Error) => void
   /** Callback when progress is reported (0-100) */
   onProgress?: (progress: number, message?: string) => void
-  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  /** Callback for each stream chunk (connect-based adapter mode only) */
   onChunk?: (chunk: StreamChunk) => void
 }
 

package/src/use-generate-video.ts

@@ -2,7 +2,7 @@ import { VideoGenerationClient } from '@tanstack/ai-client'
 import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
 import type { StreamChunk } from '@tanstack/ai'
 import type {
-  ConnectionAdapter,
+  ConnectConnectionAdapter,
   GenerationClientState,
   GenerationFetcher,
   InferGenerationOutput,
@@ -15,13 +15,13 @@ import type {
  * Options for the useGenerateVideo hook.
  */
 export interface UseGenerateVideoOptions<TOutput = VideoGenerateResult> {
-  /** Connection adapter for streaming transport (server handles polling) */
-  connection?: ConnectionAdapter
+  /** Connect-based adapter for streaming transport (server handles polling) */
+  connection?: ConnectConnectionAdapter
   /** Direct async function that returns a completed video result */
   fetcher?: GenerationFetcher<VideoGenerateInput, VideoGenerateResult>
   /** Unique identifier for this generation instance */
   id?: string
-  /** Additional body parameters to send with ConnectionAdapter requests */
+  /** Additional body parameters to send with connect-based adapter requests */
   body?: Record<string, any>
   /**
    * Callback when video generation completes. Can optionally return a transformed value.
@@ -39,7 +39,7 @@ export interface UseGenerateVideoOptions<TOutput = VideoGenerateResult> {
   onJobCreated?: (jobId: string) => void
   /** Callback on each status update */
   onStatusUpdate?: (status: VideoStatusInfo) => void
-  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  /** Callback for each stream chunk (connect-based adapter mode only) */
   onChunk?: (chunk: StreamChunk) => void
 }
 

package/src/use-generation.ts

@@ -2,7 +2,7 @@ import { GenerationClient } from '@tanstack/ai-client'
 import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
 import type { StreamChunk } from '@tanstack/ai'
 import type {
-  ConnectionAdapter,
+  ConnectConnectionAdapter,
   GenerationClientOptions,
   GenerationClientState,
   GenerationFetcher,
@@ -19,13 +19,13 @@ import type {
  * @template TOutput - The output type after optional transform (defaults to TResult)
  */
 export interface UseGenerationOptions<TInput, TResult, TOutput = TResult> {
-  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
-  connection?: ConnectionAdapter
+  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectConnectionAdapter
   /** Direct async function for one-shot generation (no streaming protocol needed) */
   fetcher?: GenerationFetcher<TInput, TResult>
   /** Unique identifier for this generation instance */
   id?: string
-  /** Additional body parameters to send with ConnectionAdapter requests */
+  /** Additional body parameters to send with connect-based adapter requests */
   body?: Record<string, any>
   /**
    * Callback when a result is received. Can optionally return a transformed value.
@@ -39,7 +39,7 @@ export interface UseGenerationOptions<TInput, TResult, TOutput = TResult> {
   onError?: (error: Error) => void
   /** Callback when progress is reported (0-100) */
   onProgress?: (progress: number, message?: string) => void
-  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  /** Callback for each stream chunk (connect-based adapter mode only) */
   onChunk?: (chunk: StreamChunk) => void
 }
 

package/src/use-realtime-chat.ts

@@ -0,0 +1,277 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { RealtimeClient } from '@tanstack/ai-client'
+import type {
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeStatus,
+} from '@tanstack/ai'
+import type {
+  UseRealtimeChatOptions,
+  UseRealtimeChatReturn,
+} from './realtime-types'
+
+// Empty frequency data for when client is not connected
+const emptyFrequencyData = new Uint8Array(128)
+const emptyTimeDomainData = new Uint8Array(128).fill(128)
+
+/**
+ * React hook for realtime voice conversations.
+ *
+ * Provides a simple interface for voice-to-voice AI interactions
+ * with support for multiple providers (OpenAI, ElevenLabs, etc.).
+ *
+ * @param options - Configuration options including adapter and callbacks
+ * @returns Hook return value with state and control methods
+ *
+ * @example
+ * ```typescript
+ * import { useRealtimeChat } from '@tanstack/ai-react'
+ * import { openaiRealtime } from '@tanstack/ai-openai'
+ *
+ * function VoiceChat() {
+ *   const {
+ *     status,
+ *     mode,
+ *     messages,
+ *     connect,
+ *     disconnect,
+ *     inputLevel,
+ *     outputLevel,
+ *   } = useRealtimeChat({
+ *     getToken: () => fetch('/api/realtime-token').then(r => r.json()),
+ *     adapter: openaiRealtime(),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <p>Status: {status}</p>
+ *       <p>Mode: {mode}</p>
+ *       <button onClick={status === 'idle' ? connect : disconnect}>
+ *         {status === 'idle' ? 'Start' : 'Stop'}
+ *       </button>
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useRealtimeChat(
+  options: UseRealtimeChatOptions,
+): UseRealtimeChatReturn {
+  // State
+  const [status, setStatus] = useState<RealtimeStatus>('idle')
+  const [mode, setMode] = useState<RealtimeMode>('idle')
+  const [messages, setMessages] = useState<Array<RealtimeMessage>>([])
+  const [pendingUserTranscript, setPendingUserTranscript] = useState<
+    string | null
+  >(null)
+  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState<
+    string | null
+  >(null)
+  const [error, setError] = useState<Error | null>(null)
+  const [inputLevel, setInputLevel] = useState(0)
+  const [outputLevel, setOutputLevel] = useState(0)
+  const [vadMode, setVADModeState] = useState<'server' | 'semantic' | 'manual'>(
+    options.vadMode ?? 'server',
+  )
+
+  // Refs
+  const clientRef = useRef<RealtimeClient | null>(null)
+  const optionsRef = useRef(options)
+  optionsRef.current = options
+  const animationFrameRef = useRef<number | null>(null)
+
+  // Create client instance - use ref to ensure we reuse the same instance
+  // This handles React StrictMode double-rendering
+  if (!clientRef.current) {
+    clientRef.current = new RealtimeClient({
+      getToken: optionsRef.current.getToken,
+      adapter: optionsRef.current.adapter,
+      tools: optionsRef.current.tools,
+      instructions: optionsRef.current.instructions,
+      voice: optionsRef.current.voice,
+      autoPlayback: optionsRef.current.autoPlayback,
+      autoCapture: optionsRef.current.autoCapture,
+      vadMode: optionsRef.current.vadMode,
+      outputModalities: optionsRef.current.outputModalities,
+      temperature: optionsRef.current.temperature,
+      maxOutputTokens: optionsRef.current.maxOutputTokens,
+      semanticEagerness: optionsRef.current.semanticEagerness,
+      onStatusChange: (newStatus) => {
+        setStatus(newStatus)
+      },
+      onModeChange: (newMode) => {
+        setMode(newMode)
+        optionsRef.current.onModeChange?.(newMode)
+      },
+      onMessage: (message) => {
+        setMessages((prev) => [...prev, message])
+        optionsRef.current.onMessage?.(message)
+      },
+      onError: (err) => {
+        setError(err)
+        optionsRef.current.onError?.(err)
+      },
+      onConnect: () => {
+        setError(null)
+        optionsRef.current.onConnect?.()
+      },
+      onDisconnect: () => {
+        optionsRef.current.onDisconnect?.()
+      },
+      onInterrupted: () => {
+        setPendingAssistantTranscript(null)
+        optionsRef.current.onInterrupted?.()
+      },
+    })
+
+    // Subscribe to state changes for transcripts
+    clientRef.current.onStateChange((state) => {
+      setPendingUserTranscript(state.pendingUserTranscript)
+      setPendingAssistantTranscript(state.pendingAssistantTranscript)
+    })
+  }
+
+  const client = clientRef.current
+
+  // Audio level animation loop
+  useEffect(() => {
+    function updateLevels() {
+      if (clientRef.current?.audio) {
+        setInputLevel(clientRef.current.audio.inputLevel)
+        setOutputLevel(clientRef.current.audio.outputLevel)
+      }
+      animationFrameRef.current = requestAnimationFrame(updateLevels)
+    }
+
+    if (status === 'connected') {
+      updateLevels()
+    }
+
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current)
+        animationFrameRef.current = null
+      }
+    }
+  }, [status])
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      clientRef.current?.destroy()
+    }
+  }, [])
+
+  // Connection methods
+  const connect = useCallback(async () => {
+    setError(null)
+    setMessages([])
+    setPendingUserTranscript(null)
+    setPendingAssistantTranscript(null)
+    await client.connect()
+  }, [client])
+
+  const disconnect = useCallback(async () => {
+    await client.disconnect()
+  }, [client])
+
+  // Voice control methods
+  const startListening = useCallback(() => {
+    client.startListening()
+  }, [client])
+
+  const stopListening = useCallback(() => {
+    client.stopListening()
+  }, [client])
+
+  const interrupt = useCallback(() => {
+    client.interrupt()
+  }, [client])
+
+  // Text input
+  const sendText = useCallback(
+    (text: string) => {
+      client.sendText(text)
+    },
+    [client],
+  )
+
+  // Image input
+  const sendImage = useCallback(
+    (imageData: string, mimeType: string) => {
+      client.sendImage(imageData, mimeType)
+    },
+    [client],
+  )
+
+  // Audio visualization
+  const getInputFrequencyData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData
+    )
+  }, [])
+
+  const getOutputFrequencyData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData
+    )
+  }, [])
+
+  const getInputTimeDomainData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData
+    )
+  }, [])
+
+  const getOutputTimeDomainData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData
+    )
+  }, [])
+
+  // VAD mode control
+  const setVADMode = useCallback(
+    (newMode: 'server' | 'semantic' | 'manual') => {
+      setVADModeState(newMode)
+      // TODO: Update session config if connected
+    },
+    [],
+  )
+
+  return {
+    // Connection state
+    status,
+    error,
+    connect,
+    disconnect,
+
+    // Conversation state
+    mode,
+    messages,
+    pendingUserTranscript,
+    pendingAssistantTranscript,
+
+    // Voice control
+    startListening,
+    stopListening,
+    interrupt,
+
+    // Text input
+    sendText,
+
+    // Image input
+    sendImage,
+
+    // Audio visualization
+    inputLevel,
+    outputLevel,
+    getInputFrequencyData,
+    getOutputFrequencyData,
+    getInputTimeDomainData,
+    getOutputTimeDomainData,
+
+    // VAD control
+    vadMode,
+    setVADMode,
+  }
+}

package/src/use-summarize.ts

@@ -1,7 +1,7 @@
 import { useGeneration } from './use-generation'
 import type { StreamChunk, SummarizationResult } from '@tanstack/ai'
 import type {
-  ConnectionAdapter,
+  ConnectConnectionAdapter,
   GenerationClientState,
   GenerationFetcher,
   InferGenerationOutput,
@@ -14,13 +14,13 @@ import type {
  * @template TOutput - The output type after optional transform (defaults to SummarizationResult)
  */
 export interface UseSummarizeOptions<TOutput = SummarizationResult> {
-  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
-  connection?: ConnectionAdapter
+  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectConnectionAdapter
   /** Direct async function for summarization */
   fetcher?: GenerationFetcher<SummarizeGenerateInput, SummarizationResult>
   /** Unique identifier for this generation instance */
   id?: string
-  /** Additional body parameters to send with ConnectionAdapter requests */
+  /** Additional body parameters to send with connect-based adapter requests */
   body?: Record<string, any>
   /**
    * Callback when summarization is complete. Can optionally return a transformed value.
@@ -34,7 +34,7 @@ export interface UseSummarizeOptions<TOutput = SummarizationResult> {
   onError?: (error: Error) => void
   /** Callback when progress is reported (0-100) */
   onProgress?: (progress: number, message?: string) => void
-  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  /** Callback for each stream chunk (connect-based adapter mode only) */
   onChunk?: (chunk: StreamChunk) => void
 }
 

package/src/use-transcription.ts

@@ -1,7 +1,7 @@
 import { useGeneration } from './use-generation'
 import type { StreamChunk, TranscriptionResult } from '@tanstack/ai'
 import type {
-  ConnectionAdapter,
+  ConnectConnectionAdapter,
   GenerationClientState,
   GenerationFetcher,
   InferGenerationOutput,
@@ -14,13 +14,13 @@ import type {
  * @template TOutput - The output type after optional transform (defaults to TranscriptionResult)
  */
 export interface UseTranscriptionOptions<TOutput = TranscriptionResult> {
-  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
-  connection?: ConnectionAdapter
+  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectConnectionAdapter
   /** Direct async function for transcription */
   fetcher?: GenerationFetcher<TranscriptionGenerateInput, TranscriptionResult>
   /** Unique identifier for this generation instance */
   id?: string
-  /** Additional body parameters to send with ConnectionAdapter requests */
+  /** Additional body parameters to send with connect-based adapter requests */
   body?: Record<string, any>
   /**
    * Callback when transcription is complete. Can optionally return a transformed value.
@@ -34,7 +34,7 @@ export interface UseTranscriptionOptions<TOutput = TranscriptionResult> {
   onError?: (error: Error) => void
   /** Callback when progress is reported (0-100) */
   onProgress?: (progress: number, message?: string) => void
-  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  /** Callback for each stream chunk (connect-based adapter mode only) */
   onChunk?: (chunk: StreamChunk) => void
 }