@tanstack/ai-react 0.6.3 → 0.7.1

package/src/use-generation.ts

@@ -0,0 +1,180 @@
+import { GenerationClient } from '@tanstack/ai-client'
+import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
+import type { StreamChunk } from '@tanstack/ai'
+import type {
+  ConnectionAdapter,
+  GenerationClientOptions,
+  GenerationClientState,
+  GenerationFetcher,
+  InferGenerationOutput,
+} from '@tanstack/ai-client'
+
+/**
+ * Options for the useGeneration hook.
+ *
+ * Accepts either a `connection` (streaming transport) or a `fetcher` (direct async call).
+ *
+ * @template TInput - The input type for the generation request
+ * @template TResult - The result type returned by the generation
+ * @template TOutput - The output type after optional transform (defaults to TResult)
+ */
+export interface UseGenerationOptions<TInput, TResult, TOutput = TResult> {
+  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectionAdapter
+  /** Direct async function for one-shot generation (no streaming protocol needed) */
+  fetcher?: GenerationFetcher<TInput, TResult>
+  /** Unique identifier for this generation instance */
+  id?: string
+  /** Additional body parameters to send with ConnectionAdapter requests */
+  body?: Record<string, any>
+  /**
+   * Callback when a result is received. Can optionally return a transformed value.
+   *
+   * - Return a non-null value to transform and store it as the result
+   * - Return `null` to keep the previous result unchanged
+   * - Return nothing (`void`) to store the raw result as-is
+   */
+  onResult?: (result: TResult) => TOutput | null | void
+  /** Callback when an error occurs */
+  onError?: (error: Error) => void
+  /** Callback when progress is reported (0-100) */
+  onProgress?: (progress: number, message?: string) => void
+  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  onChunk?: (chunk: StreamChunk) => void
+}
+
+/**
+ * Return type for the useGeneration hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseGenerationReturn<TOutput> {
+  /** Trigger a generation request */
+  generate: (input: Record<string, any>) => Promise<void>
+  /** The generation result, or null if not yet generated */
+  result: TOutput | null
+  /** Whether a generation is currently in progress */
+  isLoading: boolean
+  /** Current error, if any */
+  error: Error | undefined
+  /** Current state of the generation client */
+  status: GenerationClientState
+  /** Abort the current generation */
+  stop: () => void
+  /** Clear result, error, and return to idle */
+  reset: () => void
+}
+
+/**
+ * Generic React hook for one-shot generation tasks.
+ *
+ * This is the base hook used by `useGenerateImage`, `useGenerateSpeech`,
+ * `useTranscription`, and `useSummarize`. You can also use it directly
+ * for custom generation types.
+ *
+ * @template TInput - The input type for the generation request
+ * @template TResult - The result type returned by the generation
+ *
+ * @example
+ * ```tsx
+ * const { generate, result, isLoading } = useGeneration<MyInput, MyResult>({
+ *   connection: fetchServerSentEvents('/api/generate/custom'),
+ * })
+ *
+ * await generate({ prompt: 'Hello' })
+ * ```
+ */
+export function useGeneration<
+  TInput extends Record<string, any>,
+  TResult,
+  TOnResult extends ((result: TResult) => any) | undefined = undefined,
+>(
+  options: Omit<UseGenerationOptions<TInput, TResult>, 'onResult'> & {
+    onResult?: TOnResult
+  },
+): UseGenerationReturn<InferGenerationOutput<TResult, TOnResult>> {
+  type TOutput = InferGenerationOutput<TResult, TOnResult>
+  const hookId = useId()
+  const clientId = options.id || hookId
+
+  const [result, setResult] = useState<TOutput | null>(null)
+  const [isLoading, setIsLoading] = useState(false)
+  const [error, setError] = useState<Error | undefined>(undefined)
+  const [status, setStatus] = useState<GenerationClientState>('idle')
+
+  const optionsRef = useRef(options)
+  optionsRef.current = options
+
+  const client = useMemo(() => {
+    const opts = optionsRef.current
+
+    const clientOptions: GenerationClientOptions<TInput, TResult, TOutput> = {
+      id: clientId,
+      body: opts.body,
+      onResult: (r: TResult) => optionsRef.current.onResult?.(r),
+      onError: (e: Error) => optionsRef.current.onError?.(e),
+      onProgress: (p: number, m?: string) =>
+        optionsRef.current.onProgress?.(p, m),
+      onChunk: (c: StreamChunk) => optionsRef.current.onChunk?.(c),
+      onResultChange: setResult,
+      onLoadingChange: setIsLoading,
+      onErrorChange: setError,
+      onStatusChange: setStatus,
+    }
+
+    if (opts.connection) {
+      return new GenerationClient<TInput, TResult, TOutput>({
+        ...clientOptions,
+        connection: opts.connection,
+      })
+    }
+
+    if (opts.fetcher) {
+      return new GenerationClient<TInput, TResult, TOutput>({
+        ...clientOptions,
+        fetcher: opts.fetcher,
+      })
+    }
+
+    throw new Error(
+      'useGeneration requires either a connection or fetcher option',
+    )
+  }, [clientId])
+
+  // Sync body changes without recreating client
+  useEffect(() => {
+    client.updateOptions({ body: options.body })
+  }, [client, options.body])
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      client.stop()
+    }
+  }, [client])
+
+  const generate = useCallback(
+    async (input: TInput) => {
+      await client.generate(input)
+    },
+    [client],
+  )
+
+  const stop = useCallback(() => {
+    client.stop()
+  }, [client])
+
+  const reset = useCallback(() => {
+    client.reset()
+  }, [client])
+
+  return {
+    generate: generate as (input: Record<string, any>) => Promise<void>,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset,
+  }
+}

package/src/use-realtime-chat.ts

@@ -0,0 +1,277 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { RealtimeClient } from '@tanstack/ai-client'
+import type {
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeStatus,
+} from '@tanstack/ai'
+import type {
+  UseRealtimeChatOptions,
+  UseRealtimeChatReturn,
+} from './realtime-types'
+
+// Empty frequency data for when client is not connected
+const emptyFrequencyData = new Uint8Array(128)
+const emptyTimeDomainData = new Uint8Array(128).fill(128)
+
+/**
+ * React hook for realtime voice conversations.
+ *
+ * Provides a simple interface for voice-to-voice AI interactions
+ * with support for multiple providers (OpenAI, ElevenLabs, etc.).
+ *
+ * @param options - Configuration options including adapter and callbacks
+ * @returns Hook return value with state and control methods
+ *
+ * @example
+ * ```typescript
+ * import { useRealtimeChat } from '@tanstack/ai-react'
+ * import { openaiRealtime } from '@tanstack/ai-openai'
+ *
+ * function VoiceChat() {
+ *   const {
+ *     status,
+ *     mode,
+ *     messages,
+ *     connect,
+ *     disconnect,
+ *     inputLevel,
+ *     outputLevel,
+ *   } = useRealtimeChat({
+ *     getToken: () => fetch('/api/realtime-token').then(r => r.json()),
+ *     adapter: openaiRealtime(),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <p>Status: {status}</p>
+ *       <p>Mode: {mode}</p>
+ *       <button onClick={status === 'idle' ? connect : disconnect}>
+ *         {status === 'idle' ? 'Start' : 'Stop'}
+ *       </button>
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useRealtimeChat(
+  options: UseRealtimeChatOptions,
+): UseRealtimeChatReturn {
+  // State
+  const [status, setStatus] = useState<RealtimeStatus>('idle')
+  const [mode, setMode] = useState<RealtimeMode>('idle')
+  const [messages, setMessages] = useState<Array<RealtimeMessage>>([])
+  const [pendingUserTranscript, setPendingUserTranscript] = useState<
+    string | null
+  >(null)
+  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState<
+    string | null
+  >(null)
+  const [error, setError] = useState<Error | null>(null)
+  const [inputLevel, setInputLevel] = useState(0)
+  const [outputLevel, setOutputLevel] = useState(0)
+  const [vadMode, setVADModeState] = useState<'server' | 'semantic' | 'manual'>(
+    options.vadMode ?? 'server',
+  )
+
+  // Refs
+  const clientRef = useRef<RealtimeClient | null>(null)
+  const optionsRef = useRef(options)
+  optionsRef.current = options
+  const animationFrameRef = useRef<number | null>(null)
+
+  // Create client instance - use ref to ensure we reuse the same instance
+  // This handles React StrictMode double-rendering
+  if (!clientRef.current) {
+    clientRef.current = new RealtimeClient({
+      getToken: optionsRef.current.getToken,
+      adapter: optionsRef.current.adapter,
+      tools: optionsRef.current.tools,
+      instructions: optionsRef.current.instructions,
+      voice: optionsRef.current.voice,
+      autoPlayback: optionsRef.current.autoPlayback,
+      autoCapture: optionsRef.current.autoCapture,
+      vadMode: optionsRef.current.vadMode,
+      outputModalities: optionsRef.current.outputModalities,
+      temperature: optionsRef.current.temperature,
+      maxOutputTokens: optionsRef.current.maxOutputTokens,
+      semanticEagerness: optionsRef.current.semanticEagerness,
+      onStatusChange: (newStatus) => {
+        setStatus(newStatus)
+      },
+      onModeChange: (newMode) => {
+        setMode(newMode)
+        optionsRef.current.onModeChange?.(newMode)
+      },
+      onMessage: (message) => {
+        setMessages((prev) => [...prev, message])
+        optionsRef.current.onMessage?.(message)
+      },
+      onError: (err) => {
+        setError(err)
+        optionsRef.current.onError?.(err)
+      },
+      onConnect: () => {
+        setError(null)
+        optionsRef.current.onConnect?.()
+      },
+      onDisconnect: () => {
+        optionsRef.current.onDisconnect?.()
+      },
+      onInterrupted: () => {
+        setPendingAssistantTranscript(null)
+        optionsRef.current.onInterrupted?.()
+      },
+    })
+
+    // Subscribe to state changes for transcripts
+    clientRef.current.onStateChange((state) => {
+      setPendingUserTranscript(state.pendingUserTranscript)
+      setPendingAssistantTranscript(state.pendingAssistantTranscript)
+    })
+  }
+
+  const client = clientRef.current
+
+  // Audio level animation loop
+  useEffect(() => {
+    function updateLevels() {
+      if (clientRef.current?.audio) {
+        setInputLevel(clientRef.current.audio.inputLevel)
+        setOutputLevel(clientRef.current.audio.outputLevel)
+      }
+      animationFrameRef.current = requestAnimationFrame(updateLevels)
+    }
+
+    if (status === 'connected') {
+      updateLevels()
+    }
+
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current)
+        animationFrameRef.current = null
+      }
+    }
+  }, [status])
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      clientRef.current?.destroy()
+    }
+  }, [])
+
+  // Connection methods
+  const connect = useCallback(async () => {
+    setError(null)
+    setMessages([])
+    setPendingUserTranscript(null)
+    setPendingAssistantTranscript(null)
+    await client.connect()
+  }, [client])
+
+  const disconnect = useCallback(async () => {
+    await client.disconnect()
+  }, [client])
+
+  // Voice control methods
+  const startListening = useCallback(() => {
+    client.startListening()
+  }, [client])
+
+  const stopListening = useCallback(() => {
+    client.stopListening()
+  }, [client])
+
+  const interrupt = useCallback(() => {
+    client.interrupt()
+  }, [client])
+
+  // Text input
+  const sendText = useCallback(
+    (text: string) => {
+      client.sendText(text)
+    },
+    [client],
+  )
+
+  // Image input
+  const sendImage = useCallback(
+    (imageData: string, mimeType: string) => {
+      client.sendImage(imageData, mimeType)
+    },
+    [client],
+  )
+
+  // Audio visualization
+  const getInputFrequencyData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData
+    )
+  }, [])
+
+  const getOutputFrequencyData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData
+    )
+  }, [])
+
+  const getInputTimeDomainData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData
+    )
+  }, [])
+
+  const getOutputTimeDomainData = useCallback(() => {
+    return (
+      clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData
+    )
+  }, [])
+
+  // VAD mode control
+  const setVADMode = useCallback(
+    (newMode: 'server' | 'semantic' | 'manual') => {
+      setVADModeState(newMode)
+      // TODO: Update session config if connected
+    },
+    [],
+  )
+
+  return {
+    // Connection state
+    status,
+    error,
+    connect,
+    disconnect,
+
+    // Conversation state
+    mode,
+    messages,
+    pendingUserTranscript,
+    pendingAssistantTranscript,
+
+    // Voice control
+    startListening,
+    stopListening,
+    interrupt,
+
+    // Text input
+    sendText,
+
+    // Image input
+    sendImage,
+
+    // Audio visualization
+    inputLevel,
+    outputLevel,
+    getInputFrequencyData,
+    getOutputFrequencyData,
+    getInputTimeDomainData,
+    getOutputTimeDomainData,
+
+    // VAD control
+    vadMode,
+    setVADMode,
+  }
+}

package/src/use-summarize.ts

@@ -0,0 +1,114 @@
+import { useGeneration } from './use-generation'
+import type { StreamChunk, SummarizationResult } from '@tanstack/ai'
+import type {
+  ConnectionAdapter,
+  GenerationClientState,
+  GenerationFetcher,
+  InferGenerationOutput,
+  SummarizeGenerateInput,
+} from '@tanstack/ai-client'
+
+/**
+ * Options for the useSummarize hook.
+ *
+ * @template TOutput - The output type after optional transform (defaults to SummarizationResult)
+ */
+export interface UseSummarizeOptions<TOutput = SummarizationResult> {
+  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectionAdapter
+  /** Direct async function for summarization */
+  fetcher?: GenerationFetcher<SummarizeGenerateInput, SummarizationResult>
+  /** Unique identifier for this generation instance */
+  id?: string
+  /** Additional body parameters to send with ConnectionAdapter requests */
+  body?: Record<string, any>
+  /**
+   * Callback when summarization is complete. Can optionally return a transformed value.
+   *
+   * - Return a non-null value to transform and store it as the result
+   * - Return `null` to keep the previous result unchanged
+   * - Return nothing (`void`) to store the raw result as-is
+   */
+  onResult?: (result: SummarizationResult) => TOutput | null | void
+  /** Callback when an error occurs */
+  onError?: (error: Error) => void
+  /** Callback when progress is reported (0-100) */
+  onProgress?: (progress: number, message?: string) => void
+  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  onChunk?: (chunk: StreamChunk) => void
+}
+
+/**
+ * Return type for the useSummarize hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseSummarizeReturn<TOutput = SummarizationResult> {
+  /** Trigger summarization */
+  generate: (input: SummarizeGenerateInput) => Promise<void>
+  /** The summarization result, or null */
+  result: TOutput | null
+  /** Whether summarization is in progress */
+  isLoading: boolean
+  /** Current error, if any */
+  error: Error | undefined
+  /** Current state of the generation */
+  status: GenerationClientState
+  /** Abort the current summarization */
+  stop: () => void
+  /** Clear result, error, and return to idle */
+  reset: () => void
+}
+
+/**
+ * React hook for summarizing text using AI models.
+ *
+ * @example
+ * ```tsx
+ * import { useSummarize } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function Summarizer() {
+ *   const { generate, result, isLoading } = useSummarize({
+ *     connection: fetchServerSentEvents('/api/summarize'),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <button onClick={() => generate({
+ *         text: 'Long article text...',
+ *         style: 'bullet-points',
+ *         maxLength: 200,
+ *       })}>
+ *         Summarize
+ *       </button>
+ *       {isLoading && <p>Summarizing...</p>}
+ *       {result && <p>{result.summary}</p>}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useSummarize<
+  TOnResult extends ((result: SummarizationResult) => any) | undefined =
+    undefined,
+>(
+  options: Omit<UseSummarizeOptions, 'onResult'> & {
+    onResult?: TOnResult
+  },
+): UseSummarizeReturn<InferGenerationOutput<SummarizationResult, TOnResult>> {
+  const { generate, result, isLoading, error, status, stop, reset } =
+    useGeneration<SummarizeGenerateInput, SummarizationResult, TOnResult>(
+      options,
+    )
+
+  return {
+    generate: generate as (input: SummarizeGenerateInput) => Promise<void>,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset,
+  }
+}

package/src/use-transcription.ts

@@ -0,0 +1,121 @@
+import { useGeneration } from './use-generation'
+import type { StreamChunk, TranscriptionResult } from '@tanstack/ai'
+import type {
+  ConnectionAdapter,
+  GenerationClientState,
+  GenerationFetcher,
+  InferGenerationOutput,
+  TranscriptionGenerateInput,
+} from '@tanstack/ai-client'
+
+/**
+ * Options for the useTranscription hook.
+ *
+ * @template TOutput - The output type after optional transform (defaults to TranscriptionResult)
+ */
+export interface UseTranscriptionOptions<TOutput = TranscriptionResult> {
+  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectionAdapter
+  /** Direct async function for transcription */
+  fetcher?: GenerationFetcher<TranscriptionGenerateInput, TranscriptionResult>
+  /** Unique identifier for this generation instance */
+  id?: string
+  /** Additional body parameters to send with ConnectionAdapter requests */
+  body?: Record<string, any>
+  /**
+   * Callback when transcription is complete. Can optionally return a transformed value.
+   *
+   * - Return a non-null value to transform and store it as the result
+   * - Return `null` to keep the previous result unchanged
+   * - Return nothing (`void`) to store the raw result as-is
+   */
+  onResult?: (result: TranscriptionResult) => TOutput | null | void
+  /** Callback when an error occurs */
+  onError?: (error: Error) => void
+  /** Callback when progress is reported (0-100) */
+  onProgress?: (progress: number, message?: string) => void
+  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  onChunk?: (chunk: StreamChunk) => void
+}
+
+/**
+ * Return type for the useTranscription hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseTranscriptionReturn<TOutput = TranscriptionResult> {
+  /** Trigger transcription */
+  generate: (input: TranscriptionGenerateInput) => Promise<void>
+  /** The transcription result, or null */
+  result: TOutput | null
+  /** Whether transcription is in progress */
+  isLoading: boolean
+  /** Current error, if any */
+  error: Error | undefined
+  /** Current state of the generation */
+  status: GenerationClientState
+  /** Abort the current transcription */
+  stop: () => void
+  /** Clear result, error, and return to idle */
+  reset: () => void
+}
+
+/**
+ * React hook for transcribing audio to text using AI models.
+ *
+ * @example
+ * ```tsx
+ * import { useTranscription } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function Transcriber() {
+ *   const { generate, result, isLoading } = useTranscription({
+ *     connection: fetchServerSentEvents('/api/transcribe'),
+ *   })
+ *
+ *   const handleFile = (e: React.ChangeEvent<HTMLInputElement>) => {
+ *     const file = e.target.files?.[0]
+ *     if (file) {
+ *       const reader = new FileReader()
+ *       reader.onload = () => {
+ *         generate({ audio: reader.result as string, language: 'en' })
+ *       }
+ *       reader.readAsDataURL(file)
+ *     }
+ *   }
+ *
+ *   return (
+ *     <div>
+ *       <input type="file" accept="audio/*" onChange={handleFile} />
+ *       {isLoading && <p>Transcribing...</p>}
+ *       {result && <p>{result.text}</p>}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useTranscription<
+  TOnResult extends ((result: TranscriptionResult) => any) | undefined =
+    undefined,
+>(
+  options: Omit<UseTranscriptionOptions, 'onResult'> & {
+    onResult?: TOnResult
+  },
+): UseTranscriptionReturn<
+  InferGenerationOutput<TranscriptionResult, TOnResult>
+> {
+  const { generate, result, isLoading, error, status, stop, reset } =
+    useGeneration<TranscriptionGenerateInput, TranscriptionResult, TOnResult>(
+      options,
+    )
+
+  return {
+    generate: generate as (input: TranscriptionGenerateInput) => Promise<void>,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset,
+  }
+}