@tanstack/ai-react 0.6.3 → 0.7.1

package/src/realtime-types.ts

@@ -0,0 +1,143 @@
+import type {
+  AnyClientTool,
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeStatus,
+  RealtimeToken,
+} from '@tanstack/ai'
+import type { RealtimeAdapter } from '@tanstack/ai-client'
+
+/**
+ * Options for the useRealtimeChat hook.
+ */
+export interface UseRealtimeChatOptions {
+  /**
+   * Function to fetch a realtime token from the server.
+   * Called on connect and when token needs refresh.
+   */
+  getToken: () => Promise<RealtimeToken>
+
+  /**
+   * The realtime adapter to use (e.g., openaiRealtime())
+   */
+  adapter: RealtimeAdapter
+
+  /**
+   * Client-side tools with execution logic
+   */
+  tools?: ReadonlyArray<AnyClientTool>
+
+  /**
+   * Auto-play assistant audio (default: true)
+   */
+  autoPlayback?: boolean
+
+  /**
+   * Request microphone access on connect (default: true)
+   */
+  autoCapture?: boolean
+
+  /**
+   * System instructions for the assistant
+   */
+  instructions?: string
+
+  /**
+   * Voice to use for audio output
+   */
+  voice?: string
+
+  /**
+   * Voice activity detection mode (default: 'server')
+   */
+  vadMode?: 'server' | 'semantic' | 'manual'
+
+  /**
+   * Output modalities for responses (e.g., ['audio', 'text'])
+   */
+  outputModalities?: Array<'audio' | 'text'>
+
+  /**
+   * Temperature for generation (provider-specific range)
+   */
+  temperature?: number
+
+  /**
+   * Maximum number of tokens in a response
+   */
+  maxOutputTokens?: number | 'inf'
+
+  /**
+   * Eagerness level for semantic VAD ('low', 'medium', 'high')
+   */
+  semanticEagerness?: 'low' | 'medium' | 'high'
+
+  // Callbacks
+  onConnect?: () => void
+  onDisconnect?: () => void
+  onError?: (error: Error) => void
+  onMessage?: (message: RealtimeMessage) => void
+  onModeChange?: (mode: RealtimeMode) => void
+  onInterrupted?: () => void
+}
+
+/**
+ * Return type for the useRealtimeChat hook.
+ */
+export interface UseRealtimeChatReturn {
+  // Connection state
+  /** Current connection status */
+  status: RealtimeStatus
+  /** Current error, if any */
+  error: Error | null
+  /** Connect to the realtime session */
+  connect: () => Promise<void>
+  /** Disconnect from the realtime session */
+  disconnect: () => Promise<void>
+
+  // Conversation state
+  /** Current mode (idle, listening, thinking, speaking) */
+  mode: RealtimeMode
+  /** Conversation messages */
+  messages: Array<RealtimeMessage>
+  /** User transcript while speaking (before finalized) */
+  pendingUserTranscript: string | null
+  /** Assistant transcript while speaking (before finalized) */
+  pendingAssistantTranscript: string | null
+
+  // Voice control
+  /** Start listening for voice input (manual VAD mode) */
+  startListening: () => void
+  /** Stop listening for voice input (manual VAD mode) */
+  stopListening: () => void
+  /** Interrupt the current assistant response */
+  interrupt: () => void
+
+  // Text input
+  /** Send a text message instead of voice */
+  sendText: (text: string) => void
+
+  // Image input
+  /** Send an image to the conversation */
+  sendImage: (imageData: string, mimeType: string) => void
+
+  // Audio visualization (0-1 normalized)
+  /** Current input (microphone) volume level */
+  inputLevel: number
+  /** Current output (speaker) volume level */
+  outputLevel: number
+  /** Get frequency data for input audio visualization */
+  getInputFrequencyData: () => Uint8Array
+  /** Get frequency data for output audio visualization */
+  getOutputFrequencyData: () => Uint8Array
+  /** Get time domain data for input waveform */
+  getInputTimeDomainData: () => Uint8Array
+  /** Get time domain data for output waveform */
+  getOutputTimeDomainData: () => Uint8Array
+
+  // VAD control
+  /** Current VAD mode */
+  vadMode: 'server' | 'semantic' | 'manual'
+  /** Change VAD mode at runtime */
+  setVADMode: (mode: 'server' | 'semantic' | 'manual') => void
+}

package/src/use-generate-image.ts

@@ -0,0 +1,117 @@
+import { useGeneration } from './use-generation'
+import type { ImageGenerationResult, StreamChunk } from '@tanstack/ai'
+import type {
+  ConnectionAdapter,
+  GenerationClientState,
+  GenerationFetcher,
+  ImageGenerateInput,
+  InferGenerationOutput,
+} from '@tanstack/ai-client'
+
+/**
+ * Options for the useGenerateImage hook.
+ *
+ * @template TOutput - The output type after optional transform (defaults to ImageGenerationResult)
+ */
+export interface UseGenerateImageOptions<TOutput = ImageGenerationResult> {
+  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectionAdapter
+  /** Direct async function for image generation */
+  fetcher?: GenerationFetcher<ImageGenerateInput, ImageGenerationResult>
+  /** Unique identifier for this generation instance */
+  id?: string
+  /** Additional body parameters to send with ConnectionAdapter requests */
+  body?: Record<string, any>
+  /**
+   * Callback when images are generated. Can optionally return a transformed value.
+   *
+   * - Return a non-null value to transform and store it as the result
+   * - Return `null` to keep the previous result unchanged
+   * - Return nothing (`void`) to store the raw result as-is
+   */
+  onResult?: (result: ImageGenerationResult) => TOutput | null | void
+  /** Callback when an error occurs */
+  onError?: (error: Error) => void
+  /** Callback when progress is reported (0-100) */
+  onProgress?: (progress: number, message?: string) => void
+  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  onChunk?: (chunk: StreamChunk) => void
+}
+
+/**
+ * Return type for the useGenerateImage hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseGenerateImageReturn<TOutput = ImageGenerationResult> {
+  /** Trigger image generation */
+  generate: (input: ImageGenerateInput) => Promise<void>
+  /** The generation result containing images, or null */
+  result: TOutput | null
+  /** Whether generation is in progress */
+  isLoading: boolean
+  /** Current error, if any */
+  error: Error | undefined
+  /** Current state of the generation */
+  status: GenerationClientState
+  /** Abort the current generation */
+  stop: () => void
+  /** Clear result, error, and return to idle */
+  reset: () => void
+}
+
+/**
+ * React hook for generating images using AI models.
+ *
+ * Supports two transport modes:
+ * - **ConnectionAdapter** — Streaming transport (SSE, HTTP stream, custom)
+ * - **Fetcher** — Direct async function call
+ *
+ * @example
+ * ```tsx
+ * import { useGenerateImage } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function ImageGenerator() {
+ *   const { generate, result, isLoading, error, reset } = useGenerateImage({
+ *     connection: fetchServerSentEvents('/api/generate/image'),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <button onClick={() => generate({ prompt: 'A sunset over mountains' })}>
+ *         Generate
+ *       </button>
+ *       {isLoading && <p>Generating...</p>}
+ *       {error && <p>Error: {error.message}</p>}
+ *       {result?.images.map((img, i) => (
+ *         <img key={i} src={img.url || `data:image/png;base64,${img.b64Json}`} />
+ *       ))}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useGenerateImage<
+  TOnResult extends ((result: ImageGenerationResult) => any) | undefined =
+    undefined,
+>(
+  options: Omit<UseGenerateImageOptions, 'onResult'> & {
+    onResult?: TOnResult
+  },
+): UseGenerateImageReturn<
+  InferGenerationOutput<ImageGenerationResult, TOnResult>
+> {
+  const { generate, result, isLoading, error, status, stop, reset } =
+    useGeneration<ImageGenerateInput, ImageGenerationResult, TOnResult>(options)
+
+  return {
+    generate: generate as (input: ImageGenerateInput) => Promise<void>,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset,
+  }
+}

package/src/use-generate-speech.ts

@@ -0,0 +1,108 @@
+import { useGeneration } from './use-generation'
+import type { StreamChunk, TTSResult } from '@tanstack/ai'
+import type {
+  ConnectionAdapter,
+  GenerationClientState,
+  GenerationFetcher,
+  InferGenerationOutput,
+  SpeechGenerateInput,
+} from '@tanstack/ai-client'
+
+/**
+ * Options for the useGenerateSpeech hook.
+ *
+ * @template TOutput - The output type after optional transform (defaults to TTSResult)
+ */
+export interface UseGenerateSpeechOptions<TOutput = TTSResult> {
+  /** Connection adapter for streaming transport (SSE, HTTP stream, custom) */
+  connection?: ConnectionAdapter
+  /** Direct async function for speech generation */
+  fetcher?: GenerationFetcher<SpeechGenerateInput, TTSResult>
+  /** Unique identifier for this generation instance */
+  id?: string
+  /** Additional body parameters to send with ConnectionAdapter requests */
+  body?: Record<string, any>
+  /**
+   * Callback when speech is generated. Can optionally return a transformed value.
+   *
+   * - Return a non-null value to transform and store it as the result
+   * - Return `null` to keep the previous result unchanged
+   * - Return nothing (`void`) to store the raw result as-is
+   */
+  onResult?: (result: TTSResult) => TOutput | null | void
+  /** Callback when an error occurs */
+  onError?: (error: Error) => void
+  /** Callback when progress is reported (0-100) */
+  onProgress?: (progress: number, message?: string) => void
+  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  onChunk?: (chunk: StreamChunk) => void
+}
+
+/**
+ * Return type for the useGenerateSpeech hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseGenerateSpeechReturn<TOutput = TTSResult> {
+  /** Trigger speech generation */
+  generate: (input: SpeechGenerateInput) => Promise<void>
+  /** The TTS result containing audio data, or null */
+  result: TOutput | null
+  /** Whether generation is in progress */
+  isLoading: boolean
+  /** Current error, if any */
+  error: Error | undefined
+  /** Current state of the generation */
+  status: GenerationClientState
+  /** Abort the current generation */
+  stop: () => void
+  /** Clear result, error, and return to idle */
+  reset: () => void
+}
+
+/**
+ * React hook for generating speech (text-to-speech) using AI models.
+ *
+ * @example
+ * ```tsx
+ * import { useGenerateSpeech } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function SpeechGenerator() {
+ *   const { generate, result, isLoading } = useGenerateSpeech({
+ *     connection: fetchServerSentEvents('/api/generate/speech'),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <button onClick={() => generate({ text: 'Hello world', voice: 'alloy' })}>
+ *         Generate Speech
+ *       </button>
+ *       {result && (
+ *         <audio src={`data:audio/${result.format};base64,${result.audio}`} controls />
+ *       )}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useGenerateSpeech<
+  TOnResult extends ((result: TTSResult) => any) | undefined = undefined,
+>(
+  options: Omit<UseGenerateSpeechOptions, 'onResult'> & {
+    onResult?: TOnResult
+  },
+): UseGenerateSpeechReturn<InferGenerationOutput<TTSResult, TOnResult>> {
+  const { generate, result, isLoading, error, status, stop, reset } =
+    useGeneration<SpeechGenerateInput, TTSResult, TOnResult>(options)
+
+  return {
+    generate: generate as (input: SpeechGenerateInput) => Promise<void>,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset,
+  }
+}

package/src/use-generate-video.ts

@@ -0,0 +1,206 @@
+import { VideoGenerationClient } from '@tanstack/ai-client'
+import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
+import type { StreamChunk } from '@tanstack/ai'
+import type {
+  ConnectionAdapter,
+  GenerationClientState,
+  GenerationFetcher,
+  InferGenerationOutput,
+  VideoGenerateInput,
+  VideoGenerateResult,
+  VideoStatusInfo,
+} from '@tanstack/ai-client'
+
+/**
+ * Options for the useGenerateVideo hook.
+ */
+export interface UseGenerateVideoOptions<TOutput = VideoGenerateResult> {
+  /** Connection adapter for streaming transport (server handles polling) */
+  connection?: ConnectionAdapter
+  /** Direct async function that returns a completed video result */
+  fetcher?: GenerationFetcher<VideoGenerateInput, VideoGenerateResult>
+  /** Unique identifier for this generation instance */
+  id?: string
+  /** Additional body parameters to send with ConnectionAdapter requests */
+  body?: Record<string, any>
+  /**
+   * Callback when video generation completes. Can optionally return a transformed value.
+   *
+   * - Return a non-null value to transform and store it as the result
+   * - Return `null` to keep the previous result unchanged
+   * - Return nothing (`void`) to store the raw result as-is
+   */
+  onResult?: (result: VideoGenerateResult) => TOutput | null | void
+  /** Callback when an error occurs */
+  onError?: (error: Error) => void
+  /** Callback when progress is reported (0-100) */
+  onProgress?: (progress: number, message?: string) => void
+  /** Callback when a video job is created */
+  onJobCreated?: (jobId: string) => void
+  /** Callback on each status update */
+  onStatusUpdate?: (status: VideoStatusInfo) => void
+  /** Callback for each stream chunk (ConnectionAdapter mode only) */
+  onChunk?: (chunk: StreamChunk) => void
+}
+
+/**
+ * Return type for the useGenerateVideo hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseGenerateVideoReturn<TOutput = VideoGenerateResult> {
+  /** Trigger video generation */
+  generate: (input: VideoGenerateInput) => Promise<void>
+  /** The final video result (with URL), or null */
+  result: TOutput | null
+  /** The current job ID, or null */
+  jobId: string | null
+  /** Current video generation status info, or null */
+  videoStatus: VideoStatusInfo | null
+  /** Whether generation/polling is in progress */
+  isLoading: boolean
+  /** Current error, if any */
+  error: Error | undefined
+  /** Current state of the generation */
+  status: GenerationClientState
+  /** Abort the current generation/polling */
+  stop: () => void
+  /** Clear all state and return to idle */
+  reset: () => void
+}
+
+/**
+ * React hook for generating videos using AI models.
+ *
+ * Video generation is asynchronous: a job is created, then polled for status
+ * until completion. This hook handles the full lifecycle.
+ *
+ * @example
+ * ```tsx
+ * import { useGenerateVideo } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function VideoGenerator() {
+ *   const { generate, result, videoStatus, isLoading } = useGenerateVideo({
+ *     connection: fetchServerSentEvents('/api/generate/video'),
+ *     onStatusUpdate: (status) => console.log(`Progress: ${status.progress}%`),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <button onClick={() => generate({ prompt: 'A flying car over a city' })}>
+ *         Generate Video
+ *       </button>
+ *       {isLoading && videoStatus && (
+ *         <p>Status: {videoStatus.status} ({videoStatus.progress}%)</p>
+ *       )}
+ *       {result && <video src={result.url} controls />}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function useGenerateVideo<
+  TOnResult extends ((result: VideoGenerateResult) => any) | undefined =
+    undefined,
+>(
+  options: Omit<UseGenerateVideoOptions, 'onResult'> & {
+    onResult?: TOnResult
+  },
+): UseGenerateVideoReturn<
+  InferGenerationOutput<VideoGenerateResult, TOnResult>
+> {
+  type TOutput = InferGenerationOutput<VideoGenerateResult, TOnResult>
+  const hookId = useId()
+  const clientId = options.id || hookId
+
+  const [result, setResult] = useState<TOutput | null>(null)
+  const [jobId, setJobId] = useState<string | null>(null)
+  const [videoStatus, setVideoStatus] = useState<VideoStatusInfo | null>(null)
+  const [isLoading, setIsLoading] = useState(false)
+  const [error, setError] = useState<Error | undefined>(undefined)
+  const [status, setStatus] = useState<GenerationClientState>('idle')
+
+  const optionsRef = useRef(options)
+  optionsRef.current = options
+
+  const client = useMemo(() => {
+    const opts = optionsRef.current
+
+    const baseOptions = {
+      id: clientId,
+      body: opts.body,
+      onResult: (r: VideoGenerateResult) => optionsRef.current.onResult?.(r),
+      onError: (e: Error) => optionsRef.current.onError?.(e),
+      onProgress: (p: number, m?: string) =>
+        optionsRef.current.onProgress?.(p, m),
+      onChunk: (c: StreamChunk) => optionsRef.current.onChunk?.(c),
+      onJobCreated: (id: string) => optionsRef.current.onJobCreated?.(id),
+      onStatusUpdate: (s: VideoStatusInfo) =>
+        optionsRef.current.onStatusUpdate?.(s),
+      onResultChange: setResult,
+      onLoadingChange: setIsLoading,
+      onErrorChange: setError,
+      onStatusChange: setStatus,
+      onJobIdChange: setJobId,
+      onVideoStatusChange: setVideoStatus,
+    }
+
+    if (opts.connection) {
+      return new VideoGenerationClient<TOutput>({
+        ...baseOptions,
+        connection: opts.connection,
+      })
+    }
+
+    if (opts.fetcher) {
+      return new VideoGenerationClient<TOutput>({
+        ...baseOptions,
+        fetcher: opts.fetcher,
+      })
+    }
+
+    throw new Error(
+      'useGenerateVideo requires either a connection or fetcher option',
+    )
+  }, [clientId])
+
+  // Sync body changes without recreating client
+  useEffect(() => {
+    client.updateOptions({ body: options.body })
+  }, [client, options.body])
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      client.stop()
+    }
+  }, [client])
+
+  const generate = useCallback(
+    async (input: VideoGenerateInput) => {
+      await client.generate(input)
+    },
+    [client],
+  )
+
+  const stop = useCallback(() => {
+    client.stop()
+  }, [client])
+
+  const reset = useCallback(() => {
+    client.reset()
+  }, [client])
+
+  return {
+    generate,
+    result,
+    jobId,
+    videoStatus,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset,
+  }
+}