@assistant-ui/mcp-docs-server 0.1.26 → 0.1.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/.docs/organized/code-examples/waterfall.md +1 -1
  2. package/.docs/organized/code-examples/with-a2a.md +1 -1
  3. package/.docs/organized/code-examples/with-ag-ui.md +2 -2
  4. package/.docs/organized/code-examples/with-ai-sdk-v6.md +3 -3
  5. package/.docs/organized/code-examples/with-artifacts.md +3 -3
  6. package/.docs/organized/code-examples/with-assistant-transport.md +1 -1
  7. package/.docs/organized/code-examples/with-chain-of-thought.md +3 -3
  8. package/.docs/organized/code-examples/with-cloud-standalone.md +3 -3
  9. package/.docs/organized/code-examples/with-cloud.md +3 -3
  10. package/.docs/organized/code-examples/with-custom-thread-list.md +3 -3
  11. package/.docs/organized/code-examples/with-elevenlabs-conversational.md +511 -0
  12. package/.docs/organized/code-examples/with-elevenlabs-scribe.md +5 -5
  13. package/.docs/organized/code-examples/with-expo.md +17 -17
  14. package/.docs/organized/code-examples/with-external-store.md +1 -1
  15. package/.docs/organized/code-examples/with-ffmpeg.md +216 -62
  16. package/.docs/organized/code-examples/with-google-adk.md +2 -2
  17. package/.docs/organized/code-examples/with-heat-graph.md +1 -1
  18. package/.docs/organized/code-examples/with-interactables.md +66 -8
  19. package/.docs/organized/code-examples/with-langgraph.md +2 -2
  20. package/.docs/organized/code-examples/with-livekit.md +591 -0
  21. package/.docs/organized/code-examples/with-parent-id-grouping.md +2 -2
  22. package/.docs/organized/code-examples/with-react-hook-form.md +3 -3
  23. package/.docs/organized/code-examples/with-react-ink.md +1 -1
  24. package/.docs/organized/code-examples/with-react-router.md +6 -6
  25. package/.docs/organized/code-examples/with-store.md +7 -2
  26. package/.docs/organized/code-examples/with-tanstack.md +3 -3
  27. package/.docs/organized/code-examples/with-tap-runtime.md +1 -1
  28. package/.docs/raw/docs/(docs)/copilots/model-context.mdx +9 -1
  29. package/.docs/raw/docs/(docs)/guides/interactables.mdx +99 -37
  30. package/.docs/raw/docs/(docs)/guides/tool-ui.mdx +29 -0
  31. package/.docs/raw/docs/(docs)/guides/voice.mdx +333 -0
  32. package/.docs/raw/docs/(reference)/api-reference/primitives/message-part.mdx +23 -0
  33. package/.docs/raw/docs/runtimes/a2a/index.mdx +4 -0
  34. package/.docs/raw/docs/runtimes/ai-sdk/v6.mdx +2 -2
  35. package/.docs/raw/docs/runtimes/assistant-transport.mdx +6 -2
  36. package/.docs/raw/docs/ui/context-display.mdx +2 -2
  37. package/.docs/raw/docs/ui/model-selector.mdx +1 -1
  38. package/.docs/raw/docs/ui/voice.mdx +172 -0
  39. package/package.json +3 -4
@@ -65,8 +65,8 @@
65
65
  "@assistant-ui/react": "workspace:*",
66
66
  "@assistant-ui/react-markdown": "workspace:*",
67
67
  "@tailwindcss/vite": "^4.2.2",
68
- "@tanstack/react-router": "^1.168.4",
69
- "@tanstack/react-start": "^1.167.8",
68
+ "@tanstack/react-router": "^1.168.10",
69
+ "@tanstack/react-start": "^1.167.16",
70
70
  "class-variance-authority": "^0.7.1",
71
71
  "clsx": "^2.1.1",
72
72
  "lucide-react": "^1.7.0",
@@ -86,7 +86,7 @@
86
86
  "@types/react-dom": "^19.2.3",
87
87
  "@vitejs/plugin-react": "^6.0.1",
88
88
  "typescript": "5.9.3",
89
- "vite": "^8.0.2"
89
+ "vite": "^8.0.3"
90
90
  }
91
91
  }
92
92
 
@@ -598,7 +598,7 @@ export default nextConfig;
598
598
  "class-variance-authority": "^0.7.1",
599
599
  "clsx": "^2.1.1",
600
600
  "lucide-react": "^1.7.0",
601
- "next": "^16.2.1",
601
+ "next": "^16.2.2",
602
602
  "react": "^19.2.4",
603
603
  "react-dom": "^19.2.4",
604
604
  "tailwind-merge": "^3.5.0"
@@ -14,17 +14,25 @@ System instructions define the base behavior and knowledge available to the assi
14
14
  ```tsx
15
15
  import {
16
16
  useAssistantInstructions,
17
+ useAssistantContext,
17
18
  makeAssistantVisible,
18
19
  } from "@assistant-ui/react";
19
20
 
20
- // Via useAssistantInstructions
21
+ // Static instructions
21
22
  useAssistantInstructions("You are a helpful assistant...");
22
23
 
24
+ // Dynamic context — callback is evaluated lazily at send-time
25
+ useAssistantContext({
26
+ getContext: () => `Current page: ${window.location.href}`,
27
+ });
28
+
23
29
  // Via makeAssistantVisible
24
30
  const ReadableComponent = makeAssistantVisible(MyComponent);
25
31
  // Automatically provides component HTML as system context
26
32
  ```
27
33
 
34
+ `useAssistantInstructions` takes a static string that re-registers when changed. `useAssistantContext` takes a callback that is evaluated fresh each time the model context is read, making it ideal for injecting frequently-changing application state without triggering re-registrations.
35
+
28
36
  ### Tools
29
37
 
30
38
  Tools are functions that the assistant can use to interact with your application. They can be provided through various mechanisms:
@@ -58,7 +58,7 @@ function MyRuntimeProvider({ children }: { children: React.ReactNode }) {
58
58
  </Callout>
59
59
 
60
60
  ```tsx
61
- import { useInteractable } from "@assistant-ui/react";
61
+ import { useAssistantInteractable, useInteractableState } from "@assistant-ui/react";
62
62
  import { z } from "zod";
63
63
 
64
64
  const taskBoardSchema = z.object({
@@ -74,11 +74,12 @@ const taskBoardSchema = z.object({
74
74
  const taskBoardInitialState = { tasks: [] };
75
75
 
76
76
  function TaskBoard() {
77
- const [state, setState] = useInteractable("taskBoard", {
77
+ const id = useAssistantInteractable("taskBoard", {
78
78
  description: "A task board showing the user's tasks",
79
79
  stateSchema: taskBoardSchema,
80
80
  initialState: taskBoardInitialState,
81
81
  });
82
+ const [state, { setState }] = useInteractableState(id, taskBoardInitialState);
82
83
 
83
84
  return (
84
85
  <div>
@@ -147,7 +148,7 @@ This is especially useful for large state objects where regenerating the entire
147
148
  You can render multiple interactables with the same `name` but different `id`s. Each gets its own update tool:
148
149
 
149
150
  ```tsx
150
- import { useInteractable } from "@assistant-ui/react";
151
+ import { useAssistantInteractable, useInteractableState } from "@assistant-ui/react";
151
152
  import { z } from "zod";
152
153
 
153
154
  const noteSchema = z.object({
@@ -159,12 +160,13 @@ const noteSchema = z.object({
159
160
  const noteInitialState = { title: "New Note", content: "", color: "yellow" as const };
160
161
 
161
162
  function NoteCard({ noteId }: { noteId: string }) {
162
- const [state] = useInteractable("note", {
163
+ useAssistantInteractable("note", {
163
164
  id: noteId,
164
165
  description: "A sticky note",
165
166
  stateSchema: noteSchema,
166
167
  initialState: noteInitialState,
167
168
  });
169
+ const [state] = useInteractableState(noteId, noteInitialState);
168
170
 
169
171
  return <div>{state.title}</div>;
170
172
  }
@@ -187,12 +189,13 @@ When multiple interactables are present, you can mark one as "selected" to tell
187
189
 
188
190
  ```tsx
189
191
  function NoteCard({ noteId }: { noteId: string }) {
190
- const [state, setState, { setSelected }] = useInteractable("note", {
192
+ useAssistantInteractable("note", {
191
193
  id: noteId,
192
194
  description: "A sticky note",
193
195
  stateSchema: noteSchema,
194
196
  initialState: noteInitialState,
195
197
  });
198
+ const [state, { setSelected }] = useInteractableState(noteId, noteInitialState);
196
199
 
197
200
  return (
198
201
  <div onClick={() => setSelected(true)}>
@@ -206,12 +209,12 @@ The AI sees `(SELECTED)` in the system prompt for the focused interactable, allo
206
209
 
207
210
  ## API Reference
208
211
 
209
- ### `useInteractable`
212
+ ### `useAssistantInteractable`
210
213
 
211
- Hook that registers an interactable and returns its state with a setter.
214
+ Registers an interactable with the AI assistant. Returns the instance id.
212
215
 
213
216
  ```tsx
214
- const [state, setState, meta] = useInteractable<TState>(name, config);
217
+ const id = useAssistantInteractable(name, config);
215
218
  ```
216
219
 
217
220
  **Parameters:**
@@ -221,43 +224,37 @@ const [state, setState, meta] = useInteractable<TState>(name, config);
221
224
  | `name` | `string` | Name for the interactable (used in tool names) |
222
225
  | `config.description` | `string` | Description shown to the AI |
223
226
  | `config.stateSchema` | `StandardSchemaV1 \| JSONSchema7` | Schema for the state (e.g., a Zod schema) |
224
- | `config.initialState` | `TState` | Initial state value |
227
+ | `config.initialState` | `unknown` | Initial state value |
225
228
  | `config.id` | `string?` | Optional unique instance ID (auto-generated if omitted) |
226
229
  | `config.selected` | `boolean?` | Whether this interactable is selected |
227
230
 
228
- **Returns:** `[state, setState, { id, setSelected }]`
231
+ **Returns:** `string` the instance id (auto-generated or provided).
229
232
 
230
- | Return | Type | Description |
231
- | --- | --- | --- |
232
- | `state` | `TState` | Current state |
233
- | `setState` | `(updater: TState \| (prev: TState) => TState) => void` | State setter (like `useState`) |
234
- | `meta.id` | `string` | The instance ID (auto-generated or provided) |
235
- | `meta.setSelected` | `(selected: boolean) => void` | Mark this interactable as selected |
236
-
237
- ### `makeInteractable`
233
+ ### `useInteractableState`
238
234
 
239
- Declarative API that creates a component which registers an interactable when mounted. Useful for static configurations.
235
+ Reads and writes the state of a registered interactable.
240
236
 
241
237
  ```tsx
242
- import { makeInteractable } from "@assistant-ui/react";
238
+ const [state, { setState, setSelected, isPending, error, flush }] = useInteractableState<TState>(id, fallback?);
239
+ ```
243
240
 
244
- const TaskBoardInteractable = makeInteractable({
245
- name: "taskBoard",
246
- description: "A task board showing the user's tasks",
247
- stateSchema: taskBoardSchema,
248
- initialState: taskBoardInitialState,
249
- });
241
+ **Parameters:**
250
242
 
251
- // Mount it anywhere renders nothing, just registers the interactable
252
- function App() {
253
- return (
254
- <>
255
- <TaskBoardInteractable />
256
- <Thread />
257
- </>
258
- );
259
- }
260
- ```
243
+ | Parameter | Type | Description |
244
+ | --- | --- | --- |
245
+ | `id` | `string` | The interactable instance id (from `useAssistantInteractable`) |
246
+ | `fallback` | `TState?` | Fallback value before the interactable is registered |
247
+
248
+ **Returns:** `[state, methods]`
249
+
250
+ | Return | Type | Description |
251
+ | --- | --- | --- |
252
+ | `state` | `TState` | Current state |
253
+ | `setState` | `(updater: TState \| (prev: TState) => TState) => void` | State setter (like `useState`) |
254
+ | `setSelected` | `(selected: boolean) => void` | Mark this interactable as selected |
255
+ | `isPending` | `boolean` | Whether a persistence save is in-flight |
256
+ | `error` | `unknown` | Error from the last failed save |
257
+ | `flush` | `() => Promise<void>` | Force an immediate persistence save |
261
258
 
262
259
  ### `Interactables`
263
260
 
@@ -271,7 +268,7 @@ const aui = useAui({
271
268
 
272
269
  ## How It Works
273
270
 
274
- When you call `useInteractable("taskBoard", config)`:
271
+ When you call `useAssistantInteractable("taskBoard", config)`:
275
272
 
276
273
  1. **Registration** — the interactable is registered in the `interactables` scope with its name, description, schema, and initial state.
277
274
  2. **Tool generation** — an `update_taskBoard` frontend tool is automatically created with a partial schema (all fields optional). For multiple instances, tools are named `update_{name}_{id}`.
@@ -280,6 +277,62 @@ When you call `useInteractable("taskBoard", config)`:
280
277
  5. **Partial merge** — only the fields the AI sends are updated; the rest are preserved.
281
278
  6. **Bidirectional updates** — when the AI calls the tool, the state updates and React re-renders. When the user updates state via `setState`, the model context is notified so the AI sees the latest state on the next turn.
282
279
 
280
+ ## Persistence
281
+
282
+ By default, interactable state is in-memory and lost on page refresh. You can add persistence by providing a save callback:
283
+
284
+ ```tsx
285
+ import { useEffect } from "react";
286
+ import { useAui, Interactables } from "@assistant-ui/react";
287
+
288
+ function MyRuntimeProvider({ children }) {
289
+ const aui = useAui({ interactables: Interactables() });
290
+
291
+ useEffect(() => {
292
+ // Set up persistence adapter
293
+ aui.interactables().setPersistenceAdapter({
294
+ save: async (state) => {
295
+ localStorage.setItem("interactables", JSON.stringify(state));
296
+ },
297
+ });
298
+
299
+ // Restore saved state on mount
300
+ const saved = localStorage.getItem("interactables");
301
+ if (saved) {
302
+ aui.interactables().importState(JSON.parse(saved));
303
+ }
304
+ }, [aui]);
305
+
306
+ return /* ... */;
307
+ }
308
+ ```
309
+
310
+ ### Sync Status
311
+
312
+ When a persistence adapter is set, `useInteractableState` exposes sync metadata:
313
+
314
+ ```tsx
315
+ const [state, { setState, isPending, error, flush }] = useInteractableState(id, fallback);
316
+
317
+ // isPending — true while a save is in-flight
318
+ // error — the error from the last failed save, if any
319
+ // flush() — force an immediate save (useful before navigation)
320
+ ```
321
+
322
+ State changes are automatically debounced (500ms) before saving. When a component unregisters, any pending save is flushed immediately.
323
+
324
+ ### Export / Import
325
+
326
+ For custom persistence strategies, use `exportState` and `importState` directly:
327
+
328
+ ```tsx
329
+ const snapshot = aui.interactables().exportState();
330
+ // => { "note-1": { name: "note", state: { title: "Hello" } }, ... }
331
+
332
+ aui.interactables().importState(snapshot);
333
+ // Imported state is picked up when components next register
334
+ ```
335
+
283
336
  ## Combining with Tools
284
337
 
285
338
  You can use `Interactables` alongside `Tools`:
@@ -290,3 +343,12 @@ const aui = useAui({
290
343
  interactables: Interactables(),
291
344
  });
292
345
  ```
346
+
347
+ ## Full Example
348
+
349
+ See the complete [with-interactables example](https://github.com/assistant-ui/assistant-ui/tree/main/examples/with-interactables) for a working implementation featuring:
350
+
351
+ - **Task Board** — single-instance interactable with a custom `manage_tasks` tool
352
+ - **Sticky Notes** — multi-instance interactables with selection and partial updates
353
+ - **localStorage persistence** — state survives page refresh via `setPersistenceAdapter`
354
+ - **Sync indicator** — spinning icon while a save is in-flight (`isPending`)
@@ -748,6 +748,35 @@ useAssistantToolUI({
748
748
  server.
749
749
  </Callout>
750
750
 
751
+ ## Per-Property Streaming Status
752
+
753
+ When rendering a tool UI, you can track which arguments have finished streaming using `useToolArgsStatus`. This must be used inside a tool-call message part context.
754
+
755
+ ```tsx
756
+ import { useToolArgsStatus } from "@assistant-ui/react";
757
+
758
+ const WeatherUI = makeAssistantToolUI({
759
+ toolName: "weather",
760
+ render: ({ args }) => {
761
+ const { status, propStatus } = useToolArgsStatus<{
762
+ location: string;
763
+ unit: string;
764
+ }>();
765
+
766
+ return (
767
+ <div>
768
+ <span className={propStatus.location === "streaming" ? "animate-pulse" : ""}>
769
+ {args.location ?? "..."}
770
+ </span>
771
+ {status === "complete" && <WeatherChart data={args} />}
772
+ </div>
773
+ );
774
+ },
775
+ });
776
+ ```
777
+
778
+ `propStatus` maps each key to `"streaming"` | `"complete"` once the key appears in the partial JSON. Keys not yet present in the stream are absent from `propStatus`.
779
+
751
780
  ## Related Guides
752
781
 
753
782
  - [Tools Guide](/docs/guides/tools) - Learn how to create and use tools with AI models
@@ -0,0 +1,333 @@
1
+ ---
2
+ title: Realtime Voice
3
+ description: Bidirectional realtime voice conversations with AI agents.
4
+ ---
5
+
6
+ import { VoiceSample } from "@/components/docs/samples/voice";
7
+
8
+ assistant-ui supports realtime bidirectional voice via the `RealtimeVoiceAdapter` interface. This enables live voice conversations where the user speaks into their microphone and the AI agent responds with audio, with transcripts appearing in the thread in real time.
9
+
10
+ <VoiceSample />
11
+
12
+ ## How It Works
13
+
14
+ Unlike [Speech Synthesis](/docs/guides/speech) (text-to-speech) and [Dictation](/docs/guides/dictation) (speech-to-text), the voice adapter handles **both directions simultaneously** — the user's microphone audio is streamed to the agent, and the agent's audio response is played back, all while transcripts are appended to the message thread.
15
+
16
+ | Feature | Adapter | Direction |
17
+ |---------|---------|-----------|
18
+ | [Speech Synthesis](/docs/guides/speech) | `SpeechSynthesisAdapter` | Text → Audio (one message at a time) |
19
+ | [Dictation](/docs/guides/dictation) | `DictationAdapter` | Audio → Text (into composer) |
20
+ | **Realtime Voice** | `RealtimeVoiceAdapter` | Audio ↔ Audio (bidirectional, live) |
21
+
22
+ ## Configuration
23
+
24
+ Pass a `RealtimeVoiceAdapter` implementation to the runtime via `adapters.voice`:
25
+
26
+ ```tsx
27
+ const runtime = useChatRuntime({
28
+ adapters: {
29
+ voice: new MyVoiceAdapter({ /* ... */ }),
30
+ },
31
+ });
32
+ ```
33
+
34
+ When a voice adapter is provided, `capabilities.voice` is automatically set to `true`.
35
+
36
+ ## Hooks
37
+
38
+ ### useVoiceState
39
+
40
+ Returns the current voice session state, or `undefined` when no session is active.
41
+
42
+ ```tsx
43
+ import { useVoiceState, useVoiceVolume } from "@assistant-ui/react";
44
+
45
+ const voiceState = useVoiceState();
46
+ // voiceState?.status.type — "starting" | "running" | "ended"
47
+ // voiceState?.isMuted — boolean
48
+ // voiceState?.mode — "listening" | "speaking"
49
+
50
+ const volume = useVoiceVolume();
51
+ // volume — number (0–1, real-time audio level via separate subscription)
52
+ ```
53
+
54
+ ### useVoiceControls
55
+
56
+ Returns methods to control the voice session.
57
+
58
+ ```tsx
59
+ import { useVoiceControls } from "@assistant-ui/react";
60
+
61
+ const { connect, disconnect, mute, unmute } = useVoiceControls();
62
+ ```
63
+
64
+ ## UI Example
65
+
66
+ ```tsx
67
+ import { useVoiceState, useVoiceControls } from "@assistant-ui/react";
68
+ import { PhoneIcon, PhoneOffIcon, MicIcon, MicOffIcon } from "lucide-react";
69
+
70
+ function VoiceControls() {
71
+ const voiceState = useVoiceState();
72
+ const { connect, disconnect, mute, unmute } = useVoiceControls();
73
+
74
+ const isRunning = voiceState?.status.type === "running";
75
+ const isStarting = voiceState?.status.type === "starting";
76
+ const isMuted = voiceState?.isMuted ?? false;
77
+
78
+ if (!isRunning && !isStarting) {
79
+ return (
80
+ <button onClick={() => connect()}>
81
+ <PhoneIcon /> Connect
82
+ </button>
83
+ );
84
+ }
85
+
86
+ return (
87
+ <>
88
+ <button onClick={() => (isMuted ? unmute() : mute())} disabled={!isRunning}>
89
+ {isMuted ? <MicOffIcon /> : <MicIcon />}
90
+ {isMuted ? "Unmute" : "Mute"}
91
+ </button>
92
+ <button onClick={() => disconnect()}>
93
+ <PhoneOffIcon /> Disconnect
94
+ </button>
95
+ </>
96
+ );
97
+ }
98
+ ```
99
+
100
+ ## Custom Adapters
101
+
102
+ Implement the `RealtimeVoiceAdapter` interface to integrate with any voice provider.
103
+
104
+ ### RealtimeVoiceAdapter Interface
105
+
106
+ ```tsx
107
+ import type { RealtimeVoiceAdapter } from "@assistant-ui/react";
108
+
109
+ class MyVoiceAdapter implements RealtimeVoiceAdapter {
110
+ connect(options: {
111
+ abortSignal?: AbortSignal;
112
+ }): RealtimeVoiceAdapter.Session {
113
+ // Establish connection to your voice service
114
+ return {
115
+ get status() { /* ... */ },
116
+ get isMuted() { /* ... */ },
117
+
118
+ disconnect: () => { /* ... */ },
119
+ mute: () => { /* ... */ },
120
+ unmute: () => { /* ... */ },
121
+
122
+ onStatusChange: (callback) => {
123
+ // Status: { type: "starting" } → { type: "running" } → { type: "ended", reason }
124
+ return () => {}; // Return unsubscribe
125
+ },
126
+
127
+ onTranscript: (callback) => {
128
+ // callback({ role: "user" | "assistant", text: "...", isFinal: true })
129
+ // Transcripts are automatically appended as messages in the thread.
130
+ return () => {};
131
+ },
132
+
133
+ // Report who is speaking (drives the VoiceOrb speaking animation)
134
+ onModeChange: (callback) => {
135
+ // callback("listening") — user's turn
136
+ // callback("speaking") — agent's turn
137
+ return () => {};
138
+ },
139
+
140
+ // Report real-time audio level (0–1) for visual feedback
141
+ onVolumeChange: (callback) => {
142
+ // callback(0.72) — drives VoiceOrb amplitude and waveform bar heights
143
+ return () => {};
144
+ },
145
+ };
146
+ }
147
+ }
148
+ ```
149
+
150
+ ### Session Lifecycle
151
+
152
+ The session status follows the same pattern as other adapters:
153
+
154
+ ```
155
+ starting → running → ended
156
+ ```
157
+
158
+ The `ended` status includes a `reason`:
159
+ - `"finished"` — session ended normally
160
+ - `"cancelled"` — session was cancelled by the user
161
+ - `"error"` — session ended due to an error (includes `error` field)
162
+
163
+ ### Mode and Volume
164
+
165
+ All adapters must implement `onModeChange` and `onVolumeChange`. If your provider doesn't support these, return a no-op unsubscribe:
166
+
167
+ - **`onModeChange`** — Reports `"listening"` (user's turn) or `"speaking"` (agent's turn). The `VoiceOrb` switches to the active speaking animation.
168
+ - **`onVolumeChange`** — Reports a real-time audio level (`0`–`1`). The `VoiceOrb` modulates its amplitude and glow, and waveform bars scale to match.
169
+
170
+ When using `createVoiceSession`, these are handled automatically — call `session.emitMode()` and `session.emitVolume()` when your provider delivers data.
171
+
172
+ ### Transcript Handling
173
+
174
+ Transcripts emitted via `onTranscript` are automatically appended to the message thread:
175
+
176
+ - **User transcripts** (`role: "user"`, `isFinal: true`) are appended as user messages.
177
+ - **Assistant transcripts** (`role: "assistant"`) are streamed into an assistant message. The message shows a "running" status until `isFinal: true` is received.
178
+
179
+ ## Example: ElevenLabs Conversational AI
180
+
181
+ [ElevenLabs Conversational AI](https://elevenlabs.io/docs/agents-platform/overview) provides realtime voice agents via WebRTC.
182
+
183
+ ### Install Dependencies
184
+
185
+ ```bash
186
+ npm install @elevenlabs/client
187
+ ```
188
+
189
+ ### Adapter
190
+
191
+ ```tsx title="lib/elevenlabs-voice-adapter.ts"
192
+ import type { RealtimeVoiceAdapter, Unsubscribe } from "@assistant-ui/react";
193
+ import { VoiceConversation } from "@elevenlabs/client";
194
+
195
+ export class ElevenLabsVoiceAdapter implements RealtimeVoiceAdapter {
196
+ private _agentId: string;
197
+
198
+ constructor(options: { agentId: string }) {
199
+ this._agentId = options.agentId;
200
+ }
201
+
202
+ connect(options: {
203
+ abortSignal?: AbortSignal;
204
+ }): RealtimeVoiceAdapter.Session {
205
+ const statusCallbacks = new Set<(s: RealtimeVoiceAdapter.Status) => void>();
206
+ const transcriptCallbacks = new Set<(t: RealtimeVoiceAdapter.TranscriptItem) => void>();
207
+ const modeCallbacks = new Set<(m: RealtimeVoiceAdapter.Mode) => void>();
208
+ const volumeCallbacks = new Set<(v: number) => void>();
209
+
210
+ let currentStatus: RealtimeVoiceAdapter.Status = { type: "starting" };
211
+ let isMuted = false;
212
+ let conversation: VoiceConversation | null = null;
213
+ let disposed = false;
214
+
215
+ const updateStatus = (status: RealtimeVoiceAdapter.Status) => {
216
+ if (disposed) return;
217
+ currentStatus = status;
218
+ for (const cb of statusCallbacks) cb(status);
219
+ };
220
+
221
+ const cleanup = () => {
222
+ disposed = true;
223
+ conversation = null;
224
+ statusCallbacks.clear();
225
+ transcriptCallbacks.clear();
226
+ modeCallbacks.clear();
227
+ volumeCallbacks.clear();
228
+ };
229
+
230
+ const session: RealtimeVoiceAdapter.Session = {
231
+ get status() { return currentStatus; },
232
+ get isMuted() { return isMuted; },
233
+ disconnect: () => { conversation?.endSession(); cleanup(); },
234
+ mute: () => { conversation?.setMicMuted(true); isMuted = true; },
235
+ unmute: () => { conversation?.setMicMuted(false); isMuted = false; },
236
+ onStatusChange: (cb): Unsubscribe => {
237
+ statusCallbacks.add(cb);
238
+ return () => statusCallbacks.delete(cb);
239
+ },
240
+ onTranscript: (cb): Unsubscribe => {
241
+ transcriptCallbacks.add(cb);
242
+ return () => transcriptCallbacks.delete(cb);
243
+ },
244
+ onModeChange: (cb): Unsubscribe => {
245
+ modeCallbacks.add(cb);
246
+ return () => modeCallbacks.delete(cb);
247
+ },
248
+ onVolumeChange: (cb): Unsubscribe => {
249
+ volumeCallbacks.add(cb);
250
+ return () => volumeCallbacks.delete(cb);
251
+ },
252
+ };
253
+
254
+ if (options.abortSignal) {
255
+ options.abortSignal.addEventListener("abort", () => {
256
+ conversation?.endSession(); cleanup();
257
+ }, { once: true });
258
+ }
259
+
260
+ const doConnect = async () => {
261
+ if (disposed) return;
262
+ try {
263
+ conversation = await VoiceConversation.startSession({
264
+ agentId: this._agentId,
265
+ onConnect: () => updateStatus({ type: "running" }),
266
+ onDisconnect: () => { updateStatus({ type: "ended", reason: "finished" }); cleanup(); },
267
+ onError: (msg) => { updateStatus({ type: "ended", reason: "error", error: new Error(msg) }); cleanup(); },
268
+ onModeChange: ({ mode }) => {
269
+ if (disposed) return;
270
+ for (const cb of modeCallbacks) cb(mode === "speaking" ? "speaking" : "listening");
271
+ },
272
+ onMessage: (msg) => {
273
+ if (disposed) return;
274
+ for (const cb of transcriptCallbacks) {
275
+ cb({ role: msg.role === "user" ? "user" : "assistant", text: msg.message, isFinal: true });
276
+ }
277
+ },
278
+ });
279
+ } catch (error) {
280
+ updateStatus({ type: "ended", reason: "error", error }); cleanup();
281
+ }
282
+ };
283
+
284
+ doConnect();
285
+ return session;
286
+ }
287
+ }
288
+ ```
289
+
290
+ ### Usage
291
+
292
+ ```tsx
293
+ import { ElevenLabsVoiceAdapter } from "@/lib/elevenlabs-voice-adapter";
294
+
295
+ const runtime = useChatRuntime({
296
+ adapters: {
297
+ voice: new ElevenLabsVoiceAdapter({
298
+ agentId: process.env.NEXT_PUBLIC_ELEVENLABS_AGENT_ID!,
299
+ }),
300
+ },
301
+ });
302
+ ```
303
+
304
+ ## Example: LiveKit
305
+
306
+ [LiveKit](https://livekit.io/) provides realtime voice via WebRTC rooms with transcription support.
307
+
308
+ ### Install Dependencies
309
+
310
+ ```bash
311
+ npm install livekit-client
312
+ ```
313
+
314
+ ### Usage
315
+
316
+ ```tsx
317
+ import { LiveKitVoiceAdapter } from "@/lib/livekit-voice-adapter";
318
+
319
+ const runtime = useChatRuntime({
320
+ adapters: {
321
+ voice: new LiveKitVoiceAdapter({
322
+ url: process.env.NEXT_PUBLIC_LIVEKIT_URL!,
323
+ token: async () => {
324
+ const res = await fetch("/api/livekit-token", { method: "POST" });
325
+ const { token } = await res.json();
326
+ return token;
327
+ },
328
+ }),
329
+ },
330
+ });
331
+ ```
332
+
333
+ See the `examples/with-livekit` directory in the repository for a complete implementation including the adapter and token endpoint.
@@ -37,6 +37,29 @@ Custom data events that can be rendered as UI at their position in the message s
37
37
 
38
38
  You can use either the explicit format `{ type: "data", name: "workflow", data: {...} }` or the shorthand `data-*` prefixed format `{ type: "data-workflow", data: {...} }`. The prefixed format is automatically converted to a `DataMessagePart` (stripping the `data-` prefix as the `name`). Unknown message part types that don't match any built-in type are silently skipped with a console warning.
39
39
 
40
+ #### Streaming Data Parts
41
+
42
+ Data parts can be sent from the server using `appendData()` on the stream controller:
43
+
44
+ ```ts
45
+ controller.appendData({
46
+ type: "data",
47
+ name: "chart",
48
+ data: { labels: ["Q1", "Q2"], values: [10, 20] },
49
+ });
50
+ ```
51
+
52
+ Register a renderer with `makeAssistantDataUI` to display data parts:
53
+
54
+ ```tsx
55
+ import { makeAssistantDataUI } from "@assistant-ui/react";
56
+
57
+ const ChartUI = makeAssistantDataUI({
58
+ name: "chart",
59
+ render: ({ data }) => <MyChart data={data} />,
60
+ });
61
+ ```
62
+
40
63
  ## Anatomy
41
64
 
42
65
  ```tsx