@tanstack/ai-react 0.15.14 → 0.16.0

package/dist/esm/use-realtime-chat.js

@@ -0,0 +1,189 @@
+import { useState, useRef, useEffect, useCallback } from "react";
+import { RealtimeClient } from "@tanstack/ai-client";
+const emptyFrequencyData = new Uint8Array(128);
+const emptyTimeDomainData = new Uint8Array(128).fill(128);
+function useRealtimeChat(options) {
+  const [status, setStatus] = useState("idle");
+  const [mode, setMode] = useState("idle");
+  const [messages, setMessages] = useState([]);
+  const [pendingUserTranscript, setPendingUserTranscript] = useState(null);
+  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState(null);
+  const [error, setError] = useState(null);
+  const [inputLevel, setInputLevel] = useState(0);
+  const [outputLevel, setOutputLevel] = useState(0);
+  const [vadMode, setVADModeState] = useState(
+    options.vadMode ?? "server"
+  );
+  const clientRef = useRef(null);
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const animationFrameRef = useRef(null);
+  if (!clientRef.current) {
+    const initial = optionsRef.current;
+    clientRef.current = new RealtimeClient({
+      getToken: initial.getToken,
+      adapter: initial.adapter,
+      ...initial.tools !== void 0 && { tools: initial.tools },
+      ...initial.instructions !== void 0 && {
+        instructions: initial.instructions
+      },
+      ...initial.voice !== void 0 && { voice: initial.voice },
+      ...initial.autoPlayback !== void 0 && {
+        autoPlayback: initial.autoPlayback
+      },
+      ...initial.autoCapture !== void 0 && {
+        autoCapture: initial.autoCapture
+      },
+      ...initial.vadMode !== void 0 && { vadMode: initial.vadMode },
+      ...initial.outputModalities !== void 0 && {
+        outputModalities: initial.outputModalities
+      },
+      ...initial.temperature !== void 0 && {
+        temperature: initial.temperature
+      },
+      ...initial.maxOutputTokens !== void 0 && {
+        maxOutputTokens: initial.maxOutputTokens
+      },
+      ...initial.semanticEagerness !== void 0 && {
+        semanticEagerness: initial.semanticEagerness
+      },
+      onStatusChange: (newStatus) => {
+        setStatus(newStatus);
+      },
+      onModeChange: (newMode) => {
+        setMode(newMode);
+        optionsRef.current.onModeChange?.(newMode);
+      },
+      onMessage: (message) => {
+        setMessages((prev) => [...prev, message]);
+        optionsRef.current.onMessage?.(message);
+      },
+      onError: (err) => {
+        setError(err);
+        optionsRef.current.onError?.(err);
+      },
+      onConnect: () => {
+        setError(null);
+        optionsRef.current.onConnect?.();
+      },
+      onDisconnect: () => {
+        optionsRef.current.onDisconnect?.();
+      },
+      onInterrupted: () => {
+        setPendingAssistantTranscript(null);
+        optionsRef.current.onInterrupted?.();
+      }
+    });
+    clientRef.current.onStateChange((state) => {
+      setPendingUserTranscript(state.pendingUserTranscript);
+      setPendingAssistantTranscript(state.pendingAssistantTranscript);
+    });
+  }
+  const client = clientRef.current;
+  useEffect(() => {
+    function updateLevels() {
+      if (clientRef.current?.audio) {
+        setInputLevel(clientRef.current.audio.inputLevel);
+        setOutputLevel(clientRef.current.audio.outputLevel);
+      }
+      animationFrameRef.current = requestAnimationFrame(updateLevels);
+    }
+    if (status === "connected") {
+      updateLevels();
+    }
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current);
+        animationFrameRef.current = null;
+      }
+    };
+  }, [status]);
+  useEffect(() => {
+    return () => {
+      clientRef.current?.destroy();
+    };
+  }, []);
+  const connect = useCallback(async () => {
+    setError(null);
+    setMessages([]);
+    setPendingUserTranscript(null);
+    setPendingAssistantTranscript(null);
+    await client.connect();
+  }, [client]);
+  const disconnect = useCallback(async () => {
+    await client.disconnect();
+  }, [client]);
+  const startListening = useCallback(() => {
+    client.startListening();
+  }, [client]);
+  const stopListening = useCallback(() => {
+    client.stopListening();
+  }, [client]);
+  const interrupt = useCallback(() => {
+    client.interrupt();
+  }, [client]);
+  const sendText = useCallback(
+    (text) => {
+      client.sendText(text);
+    },
+    [client]
+  );
+  const sendImage = useCallback(
+    (imageData, mimeType) => {
+      client.sendImage(imageData, mimeType);
+    },
+    [client]
+  );
+  const getInputFrequencyData = useCallback(() => {
+    return clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData;
+  }, []);
+  const getOutputFrequencyData = useCallback(() => {
+    return clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData;
+  }, []);
+  const getInputTimeDomainData = useCallback(() => {
+    return clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData;
+  }, []);
+  const getOutputTimeDomainData = useCallback(() => {
+    return clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData;
+  }, []);
+  const setVADMode = useCallback(
+    (newMode) => {
+      setVADModeState(newMode);
+    },
+    []
+  );
+  return {
+    // Connection state
+    status,
+    error,
+    connect,
+    disconnect,
+    // Conversation state
+    mode,
+    messages,
+    pendingUserTranscript,
+    pendingAssistantTranscript,
+    // Voice control
+    startListening,
+    stopListening,
+    interrupt,
+    // Text input
+    sendText,
+    // Image input
+    sendImage,
+    // Audio visualization
+    inputLevel,
+    outputLevel,
+    getInputFrequencyData,
+    getOutputFrequencyData,
+    getInputTimeDomainData,
+    getOutputTimeDomainData,
+    // VAD control
+    vadMode,
+    setVADMode
+  };
+}
+export {
+  useRealtimeChat
+};
+//# sourceMappingURL=use-realtime-chat.js.map

package/dist/esm/use-realtime-chat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-realtime-chat.js","sources":["../../src/use-realtime-chat.ts"],"sourcesContent":["import { useCallback, useEffect, useRef, useState } from 'react'\nimport { RealtimeClient } from '@tanstack/ai-client'\nimport type {\n  RealtimeMessage,\n  RealtimeMode,\n  RealtimeStatus,\n} from '@tanstack/ai'\nimport type {\n  UseRealtimeChatOptions,\n  UseRealtimeChatReturn,\n} from './realtime-types'\n\n// Empty frequency data for when client is not connected\nconst emptyFrequencyData = new Uint8Array(128)\nconst emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n/**\n * React hook for realtime voice conversations.\n *\n * Provides a simple interface for voice-to-voice AI interactions\n * with support for multiple providers (OpenAI, ElevenLabs, etc.).\n *\n * @param options - Configuration options including adapter and callbacks\n * @returns Hook return value with state and control methods\n *\n * @example\n * ```typescript\n * import { useRealtimeChat } from '@tanstack/ai-react'\n * import { openaiRealtime } from '@tanstack/ai-openai'\n *\n * function VoiceChat() {\n *   const {\n *     status,\n *     mode,\n *     messages,\n *     connect,\n *     disconnect,\n *     inputLevel,\n *     outputLevel,\n *   } = useRealtimeChat({\n *     getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n *     adapter: openaiRealtime(),\n *   })\n *\n *   return (\n *     <div>\n *       <p>Status: {status}</p>\n *       <p>Mode: {mode}</p>\n *       <button onClick={status === 'idle' ? connect : disconnect}>\n *         {status === 'idle' ? 'Start' : 'Stop'}\n *       </button>\n *     </div>\n *   )\n * }\n * ```\n */\nexport function useRealtimeChat(\n  options: UseRealtimeChatOptions,\n): UseRealtimeChatReturn {\n  // State\n  const [status, setStatus] = useState<RealtimeStatus>('idle')\n  const [mode, setMode] = useState<RealtimeMode>('idle')\n  const [messages, setMessages] = useState<Array<RealtimeMessage>>([])\n  const [pendingUserTranscript, setPendingUserTranscript] = useState<\n    string | null\n  >(null)\n  const [pendingAssistantTranscript, setPendingAssistantTranscript] = useState<\n    string | null\n  >(null)\n  const [error, setError] = useState<Error | null>(null)\n  const [inputLevel, setInputLevel] = useState(0)\n  const [outputLevel, setOutputLevel] = useState(0)\n  const [vadMode, setVADModeState] = useState<'server' | 'semantic' | 'manual'>(\n    options.vadMode ?? 'server',\n  )\n\n  // Refs\n  const clientRef = useRef<RealtimeClient | null>(null)\n  const optionsRef = useRef(options)\n  optionsRef.current = options\n  const animationFrameRef = useRef<number | null>(null)\n\n  // Create client instance - use ref to ensure we reuse the same instance\n  // This handles React StrictMode double-rendering\n  if (!clientRef.current) {\n    // Each optional source field is spread conditionally because the\n    // `RealtimeClientOptions` target declares strict optionals\n    // (`field?: T`) and `exactOptionalPropertyTypes` rejects passing\n    // `undefined` for absent values.\n    const initial = optionsRef.current\n    clientRef.current = new RealtimeClient({\n      getToken: initial.getToken,\n      adapter: initial.adapter,\n      ...(initial.tools !== undefined && { tools: initial.tools }),\n      ...(initial.instructions !== undefined && {\n        instructions: initial.instructions,\n      }),\n      ...(initial.voice !== undefined && { voice: initial.voice }),\n      ...(initial.autoPlayback !== undefined && {\n        autoPlayback: initial.autoPlayback,\n      }),\n      ...(initial.autoCapture !== undefined && {\n        autoCapture: initial.autoCapture,\n      }),\n      ...(initial.vadMode !== undefined && { vadMode: initial.vadMode }),\n      ...(initial.outputModalities !== undefined && {\n        outputModalities: initial.outputModalities,\n      }),\n      ...(initial.temperature !== undefined && {\n        temperature: initial.temperature,\n      }),\n      ...(initial.maxOutputTokens !== undefined && {\n        maxOutputTokens: initial.maxOutputTokens,\n      }),\n      ...(initial.semanticEagerness !== undefined && {\n        semanticEagerness: initial.semanticEagerness,\n      }),\n      onStatusChange: (newStatus) => {\n        setStatus(newStatus)\n      },\n      onModeChange: (newMode) => {\n        setMode(newMode)\n        optionsRef.current.onModeChange?.(newMode)\n      },\n      onMessage: (message) => {\n        setMessages((prev) => [...prev, message])\n        optionsRef.current.onMessage?.(message)\n      },\n      onError: (err) => {\n        setError(err)\n        optionsRef.current.onError?.(err)\n      },\n      onConnect: () => {\n        setError(null)\n        optionsRef.current.onConnect?.()\n      },\n      onDisconnect: () => {\n        optionsRef.current.onDisconnect?.()\n      },\n      onInterrupted: () => {\n        setPendingAssistantTranscript(null)\n        optionsRef.current.onInterrupted?.()\n      },\n    })\n\n    // Subscribe to state changes for transcripts\n    clientRef.current.onStateChange((state) => {\n      setPendingUserTranscript(state.pendingUserTranscript)\n      setPendingAssistantTranscript(state.pendingAssistantTranscript)\n    })\n  }\n\n  const client = clientRef.current\n\n  // Audio level animation loop\n  useEffect(() => {\n    function updateLevels() {\n      if (clientRef.current?.audio) {\n        setInputLevel(clientRef.current.audio.inputLevel)\n        setOutputLevel(clientRef.current.audio.outputLevel)\n      }\n      animationFrameRef.current = requestAnimationFrame(updateLevels)\n    }\n\n    if (status === 'connected') {\n      updateLevels()\n    }\n\n    return () => {\n      if (animationFrameRef.current) {\n        cancelAnimationFrame(animationFrameRef.current)\n        animationFrameRef.current = null\n      }\n    }\n  }, [status])\n\n  // Cleanup on unmount\n  useEffect(() => {\n    return () => {\n      clientRef.current?.destroy()\n    }\n  }, [])\n\n  // Connection methods\n  const connect = useCallback(async () => {\n    setError(null)\n    setMessages([])\n    setPendingUserTranscript(null)\n    setPendingAssistantTranscript(null)\n    await client.connect()\n  }, [client])\n\n  const disconnect = useCallback(async () => {\n    await client.disconnect()\n  }, [client])\n\n  // Voice control methods\n  const startListening = useCallback(() => {\n    client.startListening()\n  }, [client])\n\n  const stopListening = useCallback(() => {\n    client.stopListening()\n  }, [client])\n\n  const interrupt = useCallback(() => {\n    client.interrupt()\n  }, [client])\n\n  // Text input\n  const sendText = useCallback(\n    (text: string) => {\n      client.sendText(text)\n    },\n    [client],\n  )\n\n  // Image input\n  const sendImage = useCallback(\n    (imageData: string, mimeType: string) => {\n      client.sendImage(imageData, mimeType)\n    },\n    [client],\n  )\n\n  // Audio visualization\n  const getInputFrequencyData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getInputFrequencyData() ?? emptyFrequencyData\n    )\n  }, [])\n\n  const getOutputFrequencyData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getOutputFrequencyData() ?? emptyFrequencyData\n    )\n  }, [])\n\n  const getInputTimeDomainData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getInputTimeDomainData() ?? emptyTimeDomainData\n    )\n  }, [])\n\n  const getOutputTimeDomainData = useCallback(() => {\n    return (\n      clientRef.current?.audio?.getOutputTimeDomainData() ?? emptyTimeDomainData\n    )\n  }, [])\n\n  // VAD mode control\n  const setVADMode = useCallback(\n    (newMode: 'server' | 'semantic' | 'manual') => {\n      setVADModeState(newMode)\n      // TODO: Update session config if connected\n    },\n    [],\n  )\n\n  return {\n    // Connection state\n    status,\n    error,\n    connect,\n    disconnect,\n\n    // Conversation state\n    mode,\n    messages,\n    pendingUserTranscript,\n    pendingAssistantTranscript,\n\n    // Voice control\n    startListening,\n    stopListening,\n    interrupt,\n\n    // Text input\n    sendText,\n\n    // Image input\n    sendImage,\n\n    // Audio visualization\n    inputLevel,\n    outputLevel,\n    getInputFrequencyData,\n    getOutputFrequencyData,\n    getInputTimeDomainData,\n    getOutputTimeDomainData,\n\n    // VAD control\n    vadMode,\n    setVADMode,\n  }\n}\n"],"names":[],"mappings":";;AAaA,MAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,MAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AA0CjD,SAAS,gBACd,SACuB;AAEvB,QAAM,CAAC,QAAQ,SAAS,IAAI,SAAyB,MAAM;AAC3D,QAAM,CAAC,MAAM,OAAO,IAAI,SAAuB,MAAM;AACrD,QAAM,CAAC,UAAU,WAAW,IAAI,SAAiC,CAAA,CAAE;AACnE,QAAM,CAAC,uBAAuB,wBAAwB,IAAI,SAExD,IAAI;AACN,QAAM,CAAC,4BAA4B,6BAA6B,IAAI,SAElE,IAAI;AACN,QAAM,CAAC,OAAO,QAAQ,IAAI,SAAuB,IAAI;AACrD,QAAM,CAAC,YAAY,aAAa,IAAI,SAAS,CAAC;AAC9C,QAAM,CAAC,aAAa,cAAc,IAAI,SAAS,CAAC;AAChD,QAAM,CAAC,SAAS,eAAe,IAAI;AAAA,IACjC,QAAQ,WAAW;AAAA,EAAA;AAIrB,QAAM,YAAY,OAA8B,IAAI;AACpD,QAAM,aAAa,OAAO,OAAO;AACjC,aAAW,UAAU;AACrB,QAAM,oBAAoB,OAAsB,IAAI;AAIpD,MAAI,CAAC,UAAU,SAAS;AAKtB,UAAM,UAAU,WAAW;AAC3B,cAAU,UAAU,IAAI,eAAe;AAAA,MACrC,UAAU,QAAQ;AAAA,MAClB,SAAS,QAAQ;AAAA,MACjB,GAAI,QAAQ,UAAU,UAAa,EAAE,OAAO,QAAQ,MAAA;AAAA,MACpD,GAAI,QAAQ,iBAAiB,UAAa;AAAA,QACxC,cAAc,QAAQ;AAAA,MAAA;AAAA,MAExB,GAAI,QAAQ,UAAU,UAAa,EAAE,OAAO,QAAQ,MAAA;AAAA,MACpD,GAAI,QAAQ,iBAAiB,UAAa;AAAA,QACxC,cAAc,QAAQ;AAAA,MAAA;AAAA,MAExB,GAAI,QAAQ,gBAAgB,UAAa;AAAA,QACvC,aAAa,QAAQ;AAAA,MAAA;AAAA,MAEvB,GAAI,QAAQ,YAAY,UAAa,EAAE,SAAS,QAAQ,QAAA;AAAA,MACxD,GAAI,QAAQ,qBAAqB,UAAa;AAAA,QAC5C,kBAAkB,QAAQ;AAAA,MAAA;AAAA,MAE5B,GAAI,QAAQ,gBAAgB,UAAa;AAAA,QACvC,aAAa,QAAQ;AAAA,MAAA;AAAA,MAEvB,GAAI,QAAQ,oBAAoB,UAAa;AAAA,QAC3C,iBAAiB,QAAQ;AAAA,MAAA;AAAA,MAE3B,GAAI,QAAQ,sBAAsB,UAAa;AAAA,QAC7C,mBAAmB,QAAQ;AAAA,MAAA;AAAA,MAE7B,gBAAgB,CAAC,cAAc;AAC7B,kBAAU,SAAS;AAAA,MACrB;AAAA,MACA,cAAc,CAAC,YAAY;AACzB,gBAAQ,OAAO;AACf,mBAAW,QAAQ,eAAe,OAAO;AAAA,MAC3C;AAAA,MACA,WAAW,CAAC,YAAY;AACtB,oBAAY,CAAC,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC;AACxC,mBAAW,QAAQ,YAAY,OAAO;AAAA,MACxC;AAAA,MACA,SAAS,CAAC,QAAQ;AAChB,iBAAS,GAAG;AACZ,mBAAW,QAAQ,UAAU,GAAG;AAAA,MAClC;AAAA,MACA,WAAW,MAAM;AACf,iBAAS,IAAI;AACb,mBAAW,QAAQ,YAAA;AAAA,MACrB;AAAA,MACA,cAAc,MAAM;AAClB,mBAAW,QAAQ,eAAA;AAAA,MACrB;AAAA,MACA,eAAe,MAAM;AACnB,sCAA8B,IAAI;AAClC,mBAAW,QAAQ,gBAAA;AAAA,MACrB;AAAA,IAAA,CACD;AAGD,cAAU,QAAQ,cAAc,CAAC,UAAU;AACzC,+BAAyB,MAAM,qBAAqB;AACpD,oCAA8B,MAAM,0BAA0B;AAAA,IAChE,CAAC;AAAA,EACH;AAEA,QAAM,SAAS,UAAU;AAGzB,YAAU,MAAM;AACd,aAAS,eAAe;AACtB,UAAI,UAAU,SAAS,OAAO;AAC5B,sBAAc,UAAU,QAAQ,MAAM,UAAU;AAChD,uBAAe,UAAU,QAAQ,MAAM,WAAW;AAAA,MACpD;AACA,wBAAkB,UAAU,sBAAsB,YAAY;AAAA,IAChE;AAEA,QAAI,WAAW,aAAa;AAC1B,mBAAA;AAAA,IACF;AAEA,WAAO,MAAM;AACX,UAAI,kBAAkB,SAAS;AAC7B,6BAAqB,kBAAkB,OAAO;AAC9C,0BAAkB,UAAU;AAAA,MAC9B;AAAA,IACF;AAAA,EACF,GAAG,CAAC,MAAM,CAAC;AAGX,YAAU,MAAM;AACd,WAAO,MAAM;AACX,gBAAU,SAAS,QAAA;AAAA,IACrB;AAAA,EACF,GAAG,CAAA,CAAE;AAGL,QAAM,UAAU,YAAY,YAAY;AACtC,aAAS,IAAI;AACb,gBAAY,CAAA,CAAE;AACd,6BAAyB,IAAI;AAC7B,kCAA8B,IAAI;AAClC,UAAM,OAAO,QAAA;AAAA,EACf,GAAG,CAAC,MAAM,CAAC;AAEX,QAAM,aAAa,YAAY,YAAY;AACzC,UAAM,OAAO,WAAA;AAAA,EACf,GAAG,CAAC,MAAM,CAAC;AAGX,QAAM,iBAAiB,YAAY,MAAM;AACvC,WAAO,eAAA;AAAA,EACT,GAAG,CAAC,MAAM,CAAC;AAEX,QAAM,gBAAgB,YAAY,MAAM;AACtC,WAAO,cAAA;AAAA,EACT,GAAG,CAAC,MAAM,CAAC;AAEX,QAAM,YAAY,YAAY,MAAM;AAClC,WAAO,UAAA;AAAA,EACT,GAAG,CAAC,MAAM,CAAC;AAGX,QAAM,WAAW;AAAA,IACf,CAAC,SAAiB;AAChB,aAAO,SAAS,IAAI;AAAA,IACtB;AAAA,IACA,CAAC,MAAM;AAAA,EAAA;AAIT,QAAM,YAAY;AAAA,IAChB,CAAC,WAAmB,aAAqB;AACvC,aAAO,UAAU,WAAW,QAAQ;AAAA,IACtC;AAAA,IACA,CAAC,MAAM;AAAA,EAAA;AAIT,QAAM,wBAAwB,YAAY,MAAM;AAC9C,WACE,UAAU,SAAS,OAAO,sBAAA,KAA2B;AAAA,EAEzD,GAAG,CAAA,CAAE;AAEL,QAAM,yBAAyB,YAAY,MAAM;AAC/C,WACE,UAAU,SAAS,OAAO,uBAAA,KAA4B;AAAA,EAE1D,GAAG,CAAA,CAAE;AAEL,QAAM,yBAAyB,YAAY,MAAM;AAC/C,WACE,UAAU,SAAS,OAAO,uBAAA,KAA4B;AAAA,EAE1D,GAAG,CAAA,CAAE;AAEL,QAAM,0BAA0B,YAAY,MAAM;AAChD,WACE,UAAU,SAAS,OAAO,wBAAA,KAA6B;AAAA,EAE3D,GAAG,CAAA,CAAE;AAGL,QAAM,aAAa;AAAA,IACjB,CAAC,YAA8C;AAC7C,sBAAgB,OAAO;AAAA,IAEzB;AAAA,IACA,CAAA;AAAA,EAAC;AAGH,SAAO;AAAA;AAAA,IAEL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA;AAAA,IAGA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/dist/esm/use-summarize.d.ts

@@ -0,0 +1,86 @@
+import { StreamChunk, SummarizationResult } from '@tanstack/ai';
+import { AIDevtoolsDisplayOptions, ConnectConnectionAdapter, GenerationClientState, GenerationFetcher, InferGenerationOutputFromReturn, SummarizeGenerateInput } from '@tanstack/ai-client';
+/**
+ * Options for the useSummarize hook.
+ *
+ * @template TOutput - The output type after optional transform (defaults to SummarizationResult)
+ */
+export interface UseSummarizeOptions<TOutput = SummarizationResult> {
+    /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+    connection?: ConnectConnectionAdapter;
+    /** Direct async function for summarization */
+    fetcher?: GenerationFetcher<SummarizeGenerateInput, SummarizationResult>;
+    /** Unique identifier for this generation instance */
+    id?: string;
+    /** Additional body parameters to send with connect-based adapter requests */
+    body?: Record<string, any>;
+    /** Display options for TanStack AI Devtools. */
+    devtools?: AIDevtoolsDisplayOptions;
+    /**
+     * Callback when summarization is complete. Can optionally return a transformed value.
+     *
+     * - Return a non-null value to transform and store it as the result
+     * - Return `null` to keep the previous result unchanged
+     * - Return nothing (`void`) to store the raw result as-is
+     */
+    onResult?: (result: SummarizationResult) => TOutput | null | void;
+    /** Callback when an error occurs */
+    onError?: (error: Error) => void;
+    /** Callback when progress is reported (0-100) */
+    onProgress?: (progress: number, message?: string) => void;
+    /** Callback for each stream chunk (connect-based adapter mode only) */
+    onChunk?: (chunk: StreamChunk) => void;
+}
+/**
+ * Return type for the useSummarize hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseSummarizeReturn<TOutput = SummarizationResult> {
+    /** Trigger summarization */
+    generate: (input: SummarizeGenerateInput) => Promise<void>;
+    /** The summarization result, or null */
+    result: TOutput | null;
+    /** Whether summarization is in progress */
+    isLoading: boolean;
+    /** Current error, if any */
+    error: Error | undefined;
+    /** Current state of the generation */
+    status: GenerationClientState;
+    /** Abort the current summarization */
+    stop: () => void;
+    /** Clear result, error, and return to idle */
+    reset: () => void;
+}
+/**
+ * React hook for summarizing text using AI models.
+ *
+ * @example
+ * ```tsx
+ * import { useSummarize } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function Summarizer() {
+ *   const { generate, result, isLoading } = useSummarize({
+ *     connection: fetchServerSentEvents('/api/summarize'),
+ *   })
+ *
+ *   return (
+ *     <div>
+ *       <button onClick={() => generate({
+ *         text: 'Long article text...',
+ *         style: 'bullet-points',
+ *         maxLength: 200,
+ *       })}>
+ *         Summarize
+ *       </button>
+ *       {isLoading && <p>Summarizing...</p>}
+ *       {result && <p>{result.summary}</p>}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export declare function useSummarize<TTransformed = void>(options: Omit<UseSummarizeOptions, 'onResult'> & {
+    onResult?: (result: SummarizationResult) => TTransformed;
+}): UseSummarizeReturn<InferGenerationOutputFromReturn<SummarizationResult, TTransformed>>;

package/dist/esm/use-summarize.js

@@ -0,0 +1,26 @@
+import { useGeneration } from "./use-generation.js";
+function useSummarize(options) {
+  const devtools = {
+    ...options.devtools,
+    framework: "react",
+    hookName: "useSummarize",
+    outputKind: "text"
+  };
+  const { generate, result, isLoading, error, status, stop, reset } = useGeneration({
+    ...options,
+    devtools
+  });
+  return {
+    generate,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset
+  };
+}
+export {
+  useSummarize
+};
+//# sourceMappingURL=use-summarize.js.map

package/dist/esm/use-summarize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-summarize.js","sources":["../../src/use-summarize.ts"],"sourcesContent":["import { useGeneration } from './use-generation'\nimport type { StreamChunk, SummarizationResult } from '@tanstack/ai'\nimport type {\n  AIDevtoolsDisplayOptions,\n  ConnectConnectionAdapter,\n  GenerationClientState,\n  GenerationFetcher,\n  InferGenerationOutputFromReturn,\n  SummarizeGenerateInput,\n} from '@tanstack/ai-client'\n\n/**\n * Options for the useSummarize hook.\n *\n * @template TOutput - The output type after optional transform (defaults to SummarizationResult)\n */\nexport interface UseSummarizeOptions<TOutput = SummarizationResult> {\n  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */\n  connection?: ConnectConnectionAdapter\n  /** Direct async function for summarization */\n  fetcher?: GenerationFetcher<SummarizeGenerateInput, SummarizationResult>\n  /** Unique identifier for this generation instance */\n  id?: string\n  /** Additional body parameters to send with connect-based adapter requests */\n  body?: Record<string, any>\n  /** Display options for TanStack AI Devtools. */\n  devtools?: AIDevtoolsDisplayOptions\n  /**\n   * Callback when summarization is complete. Can optionally return a transformed value.\n   *\n   * - Return a non-null value to transform and store it as the result\n   * - Return `null` to keep the previous result unchanged\n   * - Return nothing (`void`) to store the raw result as-is\n   */\n  onResult?: (result: SummarizationResult) => TOutput | null | void\n  /** Callback when an error occurs */\n  onError?: (error: Error) => void\n  /** Callback when progress is reported (0-100) */\n  onProgress?: (progress: number, message?: string) => void\n  /** Callback for each stream chunk (connect-based adapter mode only) */\n  onChunk?: (chunk: StreamChunk) => void\n}\n\n/**\n * Return type for the useSummarize hook.\n *\n * @template TOutput - The output type (after optional transform)\n */\nexport interface UseSummarizeReturn<TOutput = SummarizationResult> {\n  /** Trigger summarization */\n  generate: (input: SummarizeGenerateInput) => Promise<void>\n  /** The summarization result, or null */\n  result: TOutput | null\n  /** Whether summarization is in progress */\n  isLoading: boolean\n  /** Current error, if any */\n  error: Error | undefined\n  /** Current state of the generation */\n  status: GenerationClientState\n  /** Abort the current summarization */\n  stop: () => void\n  /** Clear result, error, and return to idle */\n  reset: () => void\n}\n\n/**\n * React hook for summarizing text using AI models.\n *\n * @example\n * ```tsx\n * import { useSummarize } from '@tanstack/ai-react'\n * import { fetchServerSentEvents } from '@tanstack/ai-client'\n *\n * function Summarizer() {\n *   const { generate, result, isLoading } = useSummarize({\n *     connection: fetchServerSentEvents('/api/summarize'),\n *   })\n *\n *   return (\n *     <div>\n *       <button onClick={() => generate({\n *         text: 'Long article text...',\n *         style: 'bullet-points',\n *         maxLength: 200,\n *       })}>\n *         Summarize\n *       </button>\n *       {isLoading && <p>Summarizing...</p>}\n *       {result && <p>{result.summary}</p>}\n *     </div>\n *   )\n * }\n * ```\n */\nexport function useSummarize<TTransformed = void>(\n  options: Omit<UseSummarizeOptions, 'onResult'> & {\n    onResult?: (result: SummarizationResult) => TTransformed\n  },\n): UseSummarizeReturn<\n  InferGenerationOutputFromReturn<SummarizationResult, TTransformed>\n> {\n  const devtools = {\n    ...options.devtools,\n    framework: 'react',\n    hookName: 'useSummarize',\n    outputKind: 'text' as const,\n  }\n  const { generate, result, isLoading, error, status, stop, reset } =\n    useGeneration<SummarizeGenerateInput, SummarizationResult, TTransformed>({\n      ...options,\n      devtools,\n    })\n\n  return {\n    generate: generate as (input: SummarizeGenerateInput) => Promise<void>,\n    result,\n    isLoading,\n    error,\n    status,\n    stop,\n    reset,\n  }\n}\n"],"names":[],"mappings":";AA8FO,SAAS,aACd,SAKA;AACA,QAAM,WAAW;AAAA,IACf,GAAG,QAAQ;AAAA,IACX,WAAW;AAAA,IACX,UAAU;AAAA,IACV,YAAY;AAAA,EAAA;AAEd,QAAM,EAAE,UAAU,QAAQ,WAAW,OAAO,QAAQ,MAAM,MAAA,IACxD,cAAyE;AAAA,IACvE,GAAG;AAAA,IACH;AAAA,EAAA,CACD;AAEH,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/dist/esm/use-transcription.d.ts

@@ -0,0 +1,91 @@
+import { StreamChunk, TranscriptionResult } from '@tanstack/ai';
+import { AIDevtoolsDisplayOptions, ConnectConnectionAdapter, GenerationClientState, GenerationFetcher, InferGenerationOutputFromReturn, TranscriptionGenerateInput } from '@tanstack/ai-client';
+/**
+ * Options for the useTranscription hook.
+ *
+ * @template TOutput - The output type after optional transform (defaults to TranscriptionResult)
+ */
+export interface UseTranscriptionOptions<TOutput = TranscriptionResult> {
+    /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */
+    connection?: ConnectConnectionAdapter;
+    /** Direct async function for transcription */
+    fetcher?: GenerationFetcher<TranscriptionGenerateInput, TranscriptionResult>;
+    /** Unique identifier for this generation instance */
+    id?: string;
+    /** Additional body parameters to send with connect-based adapter requests */
+    body?: Record<string, any>;
+    /** Display options for TanStack AI Devtools. */
+    devtools?: AIDevtoolsDisplayOptions;
+    /**
+     * Callback when transcription is complete. Can optionally return a transformed value.
+     *
+     * - Return a non-null value to transform and store it as the result
+     * - Return `null` to keep the previous result unchanged
+     * - Return nothing (`void`) to store the raw result as-is
+     */
+    onResult?: (result: TranscriptionResult) => TOutput | null | void;
+    /** Callback when an error occurs */
+    onError?: (error: Error) => void;
+    /** Callback when progress is reported (0-100) */
+    onProgress?: (progress: number, message?: string) => void;
+    /** Callback for each stream chunk (connect-based adapter mode only) */
+    onChunk?: (chunk: StreamChunk) => void;
+}
+/**
+ * Return type for the useTranscription hook.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export interface UseTranscriptionReturn<TOutput = TranscriptionResult> {
+    /** Trigger transcription */
+    generate: (input: TranscriptionGenerateInput) => Promise<void>;
+    /** The transcription result, or null */
+    result: TOutput | null;
+    /** Whether transcription is in progress */
+    isLoading: boolean;
+    /** Current error, if any */
+    error: Error | undefined;
+    /** Current state of the generation */
+    status: GenerationClientState;
+    /** Abort the current transcription */
+    stop: () => void;
+    /** Clear result, error, and return to idle */
+    reset: () => void;
+}
+/**
+ * React hook for transcribing audio to text using AI models.
+ *
+ * @example
+ * ```tsx
+ * import { useTranscription } from '@tanstack/ai-react'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ *
+ * function Transcriber() {
+ *   const { generate, result, isLoading } = useTranscription({
+ *     connection: fetchServerSentEvents('/api/transcribe'),
+ *   })
+ *
+ *   const handleFile = (e: React.ChangeEvent<HTMLInputElement>) => {
+ *     const file = e.target.files?.[0]
+ *     if (file) {
+ *       const reader = new FileReader()
+ *       reader.onload = () => {
+ *         generate({ audio: reader.result as string, language: 'en' })
+ *       }
+ *       reader.readAsDataURL(file)
+ *     }
+ *   }
+ *
+ *   return (
+ *     <div>
+ *       <input type="file" accept="audio/*" onChange={handleFile} />
+ *       {isLoading && <p>Transcribing...</p>}
+ *       {result && <p>{result.text}</p>}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export declare function useTranscription<TTransformed = void>(options: Omit<UseTranscriptionOptions, 'onResult'> & {
+    onResult?: (result: TranscriptionResult) => TTransformed;
+}): UseTranscriptionReturn<InferGenerationOutputFromReturn<TranscriptionResult, TTransformed>>;

package/dist/esm/use-transcription.js

@@ -0,0 +1,23 @@
+import { useGeneration } from "./use-generation.js";
+function useTranscription(options) {
+  const devtools = {
+    ...options.devtools,
+    framework: "react",
+    hookName: "useTranscription",
+    outputKind: "text"
+  };
+  const { generate, result, isLoading, error, status, stop, reset } = useGeneration({ ...options, devtools });
+  return {
+    generate,
+    result,
+    isLoading,
+    error,
+    status,
+    stop,
+    reset
+  };
+}
+export {
+  useTranscription
+};
+//# sourceMappingURL=use-transcription.js.map

package/dist/esm/use-transcription.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-transcription.js","sources":["../../src/use-transcription.ts"],"sourcesContent":["import { useGeneration } from './use-generation'\nimport type { StreamChunk, TranscriptionResult } from '@tanstack/ai'\nimport type {\n  AIDevtoolsDisplayOptions,\n  ConnectConnectionAdapter,\n  GenerationClientState,\n  GenerationFetcher,\n  InferGenerationOutputFromReturn,\n  TranscriptionGenerateInput,\n} from '@tanstack/ai-client'\n\n/**\n * Options for the useTranscription hook.\n *\n * @template TOutput - The output type after optional transform (defaults to TranscriptionResult)\n */\nexport interface UseTranscriptionOptions<TOutput = TranscriptionResult> {\n  /** Connect-based adapter for streaming transport (SSE, HTTP stream, custom) */\n  connection?: ConnectConnectionAdapter\n  /** Direct async function for transcription */\n  fetcher?: GenerationFetcher<TranscriptionGenerateInput, TranscriptionResult>\n  /** Unique identifier for this generation instance */\n  id?: string\n  /** Additional body parameters to send with connect-based adapter requests */\n  body?: Record<string, any>\n  /** Display options for TanStack AI Devtools. */\n  devtools?: AIDevtoolsDisplayOptions\n  /**\n   * Callback when transcription is complete. Can optionally return a transformed value.\n   *\n   * - Return a non-null value to transform and store it as the result\n   * - Return `null` to keep the previous result unchanged\n   * - Return nothing (`void`) to store the raw result as-is\n   */\n  onResult?: (result: TranscriptionResult) => TOutput | null | void\n  /** Callback when an error occurs */\n  onError?: (error: Error) => void\n  /** Callback when progress is reported (0-100) */\n  onProgress?: (progress: number, message?: string) => void\n  /** Callback for each stream chunk (connect-based adapter mode only) */\n  onChunk?: (chunk: StreamChunk) => void\n}\n\n/**\n * Return type for the useTranscription hook.\n *\n * @template TOutput - The output type (after optional transform)\n */\nexport interface UseTranscriptionReturn<TOutput = TranscriptionResult> {\n  /** Trigger transcription */\n  generate: (input: TranscriptionGenerateInput) => Promise<void>\n  /** The transcription result, or null */\n  result: TOutput | null\n  /** Whether transcription is in progress */\n  isLoading: boolean\n  /** Current error, if any */\n  error: Error | undefined\n  /** Current state of the generation */\n  status: GenerationClientState\n  /** Abort the current transcription */\n  stop: () => void\n  /** Clear result, error, and return to idle */\n  reset: () => void\n}\n\n/**\n * React hook for transcribing audio to text using AI models.\n *\n * @example\n * ```tsx\n * import { useTranscription } from '@tanstack/ai-react'\n * import { fetchServerSentEvents } from '@tanstack/ai-client'\n *\n * function Transcriber() {\n *   const { generate, result, isLoading } = useTranscription({\n *     connection: fetchServerSentEvents('/api/transcribe'),\n *   })\n *\n *   const handleFile = (e: React.ChangeEvent<HTMLInputElement>) => {\n *     const file = e.target.files?.[0]\n *     if (file) {\n *       const reader = new FileReader()\n *       reader.onload = () => {\n *         generate({ audio: reader.result as string, language: 'en' })\n *       }\n *       reader.readAsDataURL(file)\n *     }\n *   }\n *\n *   return (\n *     <div>\n *       <input type=\"file\" accept=\"audio/*\" onChange={handleFile} />\n *       {isLoading && <p>Transcribing...</p>}\n *       {result && <p>{result.text}</p>}\n *     </div>\n *   )\n * }\n * ```\n */\nexport function useTranscription<TTransformed = void>(\n  options: Omit<UseTranscriptionOptions, 'onResult'> & {\n    onResult?: (result: TranscriptionResult) => TTransformed\n  },\n): UseTranscriptionReturn<\n  InferGenerationOutputFromReturn<TranscriptionResult, TTransformed>\n> {\n  const devtools = {\n    ...options.devtools,\n    framework: 'react',\n    hookName: 'useTranscription',\n    outputKind: 'text' as const,\n  }\n  const { generate, result, isLoading, error, status, stop, reset } =\n    useGeneration<\n      TranscriptionGenerateInput,\n      TranscriptionResult,\n      TTransformed\n    >({ ...options, devtools })\n\n  return {\n    generate: generate as (input: TranscriptionGenerateInput) => Promise<void>,\n    result,\n    isLoading,\n    error,\n    status,\n    stop,\n    reset,\n  }\n}\n"],"names":[],"mappings":";AAmGO,SAAS,iBACd,SAKA;AACA,QAAM,WAAW;AAAA,IACf,GAAG,QAAQ;AAAA,IACX,WAAW;AAAA,IACX,UAAU;AAAA,IACV,YAAY;AAAA,EAAA;AAEd,QAAM,EAAE,UAAU,QAAQ,WAAW,OAAO,QAAQ,MAAM,MAAA,IACxD,cAIE,EAAE,GAAG,SAAS,UAAU;AAE5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai-react",
-  "version": "0.15.14",
+  "version": "0.16.0",
   "description": "React hooks for TanStack AI streaming chat, realtime voice, structured outputs, and media generation.",
   "author": "Tanner Linsley",
   "license": "MIT",
@@ -24,6 +24,10 @@
     ".": {
       "types": "./dist/esm/index.d.ts",
       "import": "./dist/esm/index.js"
+    },
+    "./mcp-apps": {
+      "types": "./dist/esm/mcp-apps.d.ts",
+      "import": "./dist/esm/mcp-apps.js"
     }
   },
   "files": [
@@ -45,14 +49,21 @@
     "media-generation"
   ],
   "dependencies": {
-    "@tanstack/ai-client": "0.18.5"
+    "@tanstack/ai-client": "0.19.0"
   },
   "peerDependencies": {
+    "@mcp-ui/client": "^7",
     "@types/react": ">=18.0.0",
     "react": ">=18.0.0",
-    "@tanstack/ai": "^0.36.0"
+    "@tanstack/ai": "^0.38.0"
+  },
+  "peerDependenciesMeta": {
+    "@mcp-ui/client": {
+      "optional": true
+    }
   },
   "devDependencies": {
+    "@mcp-ui/client": "^7.1.1",
     "@standard-schema/spec": "^1.1.0",
     "@testing-library/react": "^16.3.0",
     "@types/react": "^19.2.7",
@@ -60,7 +71,7 @@
     "jsdom": "^27.2.0",
     "react": "^19.2.3",
     "vite": "^7.3.3",
-    "@tanstack/ai": "0.36.0"
+    "@tanstack/ai": "0.38.0"
   },
   "scripts": {
     "clean": "premove ./build ./dist",

package/src/index.ts

@@ -1,5 +1,7 @@
 export { useChat } from './use-chat'
 export { useRealtimeChat } from './use-realtime-chat'
+export { useMcpAppBridge } from './use-mcp-app-bridge'
+export type { UseMcpAppBridgeOptions } from './use-mcp-app-bridge'
 export type {
   DeepPartial,
   UseChatOptions,
@@ -52,6 +54,12 @@ export type {
   UseGenerateVideoReturn,
 } from './use-generate-video'
 
+export { useAudioRecorder } from './use-audio-recorder'
+export type {
+  UseAudioRecorderOptions,
+  UseAudioRecorderReturn,
+} from './use-audio-recorder'
+
 // Re-export from ai-client for convenience
 export {
   fetchServerSentEvents,
@@ -61,6 +69,9 @@ export {
   stream,
   rpcStream,
   createChatClientOptions,
+  createMcpAppBridge,
+  type McpAppBridge,
+  type CreateMcpAppBridgeOptions,
   type ChatFetcher,
   type ChatFetcherInput,
   type ChatFetcherOptions,

package/src/mcp-apps.tsx

@@ -0,0 +1,88 @@
+import { AppRenderer } from '@mcp-ui/client'
+import type { UIResourcePart } from '@tanstack/ai'
+import type { McpAppBridge } from '@tanstack/ai-client'
+import type { JSX } from 'react'
+
+export interface MCPAppResourceProps {
+  /** The ui-resource part from a UIMessage assistant part. */
+  part: UIResourcePart
+  /**
+   * Framework-agnostic bridge for tool calls, prompt sending, and link opening.
+   * Omit it to render the widget in display-only mode — iframe interactions
+   * that would trigger tool calls or prompts are ignored.
+   */
+  bridge?: McpAppBridge
+  /** Sandbox iframe configuration — must include the proxy page URL. */
+  sandbox: { url: URL }
+  /** Optional structured arguments forwarded to the guest UI once it's ready. */
+  toolInput?: Record<string, unknown>
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+/**
+ * Coalesce an arbitrary bridge result into the `string` required by the
+ * `CallToolResult` text content block. `JSON.stringify` is typed to return
+ * `string` but actually returns `undefined` for inputs like `undefined` or a
+ * function, so we narrow explicitly to avoid `text: undefined`.
+ */
+function resultToText(result: unknown): string {
+  if (typeof result === 'string') return result
+  // `JSON.stringify`'s lib signature claims `string`, but it returns
+  // `undefined` for `undefined`/function inputs; type the call honestly.
+  const stringify: (value: unknown) => string | undefined = JSON.stringify
+  return stringify(result) ?? 'null'
+}
+
+export function MCPAppResource(props: MCPAppResourceProps): JSX.Element {
+  const { bridge } = props
+  return (
+    <AppRenderer
+      toolName={props.part.toolName}
+      sandbox={props.sandbox}
+      html={props.part.resource.text}
+      toolResourceUri={props.part.resource.uri}
+      toolInput={props.toolInput}
+      onCallTool={
+        bridge
+          ? async ({ name, arguments: args }) => {
+              const result = await bridge.callTool({
+                serverId: props.part.serverId,
+                toolName: name,
+                args,
+              })
+              const structuredContent = isRecord(result) ? result : undefined
+              return {
+                content: [
+                  {
+                    type: 'text' as const,
+                    text: resultToText(result),
+                  },
+                ],
+                structuredContent,
+              }
+            }
+          : undefined
+      }
+      onMessage={
+        bridge
+          ? async ({ content }) => {
+              const text = content
+                .filter(
+                  (c): c is { type: 'text'; text: string } => c.type === 'text',
+                )
+                .map((c) => c.text)
+                .join('')
+              if (text) await bridge.sendPrompt(text)
+              return {}
+            }
+          : undefined
+      }
+      onOpenLink={
+        bridge ? ({ url }) => Promise.resolve(bridge.openLink(url)) : undefined
+      }
+    />
+  )
+}