@assistant-ui/mcp-docs-server 0.1.26 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/organized/code-examples/waterfall.md +1 -1
- package/.docs/organized/code-examples/with-a2a.md +1 -1
- package/.docs/organized/code-examples/with-ag-ui.md +2 -2
- package/.docs/organized/code-examples/with-ai-sdk-v6.md +3 -3
- package/.docs/organized/code-examples/with-artifacts.md +3 -3
- package/.docs/organized/code-examples/with-assistant-transport.md +1 -1
- package/.docs/organized/code-examples/with-chain-of-thought.md +3 -3
- package/.docs/organized/code-examples/with-cloud-standalone.md +3 -3
- package/.docs/organized/code-examples/with-cloud.md +3 -3
- package/.docs/organized/code-examples/with-custom-thread-list.md +3 -3
- package/.docs/organized/code-examples/with-elevenlabs-conversational.md +511 -0
- package/.docs/organized/code-examples/with-elevenlabs-scribe.md +5 -5
- package/.docs/organized/code-examples/with-expo.md +17 -17
- package/.docs/organized/code-examples/with-external-store.md +1 -1
- package/.docs/organized/code-examples/with-ffmpeg.md +216 -62
- package/.docs/organized/code-examples/with-google-adk.md +2 -2
- package/.docs/organized/code-examples/with-heat-graph.md +1 -1
- package/.docs/organized/code-examples/with-interactables.md +66 -8
- package/.docs/organized/code-examples/with-langgraph.md +2 -2
- package/.docs/organized/code-examples/with-livekit.md +591 -0
- package/.docs/organized/code-examples/with-parent-id-grouping.md +2 -2
- package/.docs/organized/code-examples/with-react-hook-form.md +3 -3
- package/.docs/organized/code-examples/with-react-ink.md +1 -1
- package/.docs/organized/code-examples/with-react-router.md +6 -6
- package/.docs/organized/code-examples/with-store.md +7 -2
- package/.docs/organized/code-examples/with-tanstack.md +3 -3
- package/.docs/organized/code-examples/with-tap-runtime.md +1 -1
- package/.docs/raw/docs/(docs)/copilots/model-context.mdx +9 -1
- package/.docs/raw/docs/(docs)/guides/interactables.mdx +99 -37
- package/.docs/raw/docs/(docs)/guides/tool-ui.mdx +29 -0
- package/.docs/raw/docs/(docs)/guides/voice.mdx +333 -0
- package/.docs/raw/docs/(reference)/api-reference/primitives/message-part.mdx +23 -0
- package/.docs/raw/docs/runtimes/a2a/index.mdx +4 -0
- package/.docs/raw/docs/runtimes/ai-sdk/v6.mdx +2 -2
- package/.docs/raw/docs/runtimes/assistant-transport.mdx +6 -2
- package/.docs/raw/docs/ui/context-display.mdx +2 -2
- package/.docs/raw/docs/ui/model-selector.mdx +1 -1
- package/.docs/raw/docs/ui/voice.mdx +172 -0
- package/package.json +3 -4
|
@@ -65,8 +65,8 @@
|
|
|
65
65
|
"@assistant-ui/react": "workspace:*",
|
|
66
66
|
"@assistant-ui/react-markdown": "workspace:*",
|
|
67
67
|
"@tailwindcss/vite": "^4.2.2",
|
|
68
|
-
"@tanstack/react-router": "^1.168.
|
|
69
|
-
"@tanstack/react-start": "^1.167.
|
|
68
|
+
"@tanstack/react-router": "^1.168.10",
|
|
69
|
+
"@tanstack/react-start": "^1.167.16",
|
|
70
70
|
"class-variance-authority": "^0.7.1",
|
|
71
71
|
"clsx": "^2.1.1",
|
|
72
72
|
"lucide-react": "^1.7.0",
|
|
@@ -86,7 +86,7 @@
|
|
|
86
86
|
"@types/react-dom": "^19.2.3",
|
|
87
87
|
"@vitejs/plugin-react": "^6.0.1",
|
|
88
88
|
"typescript": "5.9.3",
|
|
89
|
-
"vite": "^8.0.
|
|
89
|
+
"vite": "^8.0.3"
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
|
|
@@ -14,17 +14,25 @@ System instructions define the base behavior and knowledge available to the assi
|
|
|
14
14
|
```tsx
|
|
15
15
|
import {
|
|
16
16
|
useAssistantInstructions,
|
|
17
|
+
useAssistantContext,
|
|
17
18
|
makeAssistantVisible,
|
|
18
19
|
} from "@assistant-ui/react";
|
|
19
20
|
|
|
20
|
-
//
|
|
21
|
+
// Static instructions
|
|
21
22
|
useAssistantInstructions("You are a helpful assistant...");
|
|
22
23
|
|
|
24
|
+
// Dynamic context — callback is evaluated lazily at send-time
|
|
25
|
+
useAssistantContext({
|
|
26
|
+
getContext: () => `Current page: ${window.location.href}`,
|
|
27
|
+
});
|
|
28
|
+
|
|
23
29
|
// Via makeAssistantVisible
|
|
24
30
|
const ReadableComponent = makeAssistantVisible(MyComponent);
|
|
25
31
|
// Automatically provides component HTML as system context
|
|
26
32
|
```
|
|
27
33
|
|
|
34
|
+
`useAssistantInstructions` takes a static string that re-registers when changed. `useAssistantContext` takes a callback that is evaluated fresh each time the model context is read, making it ideal for injecting frequently-changing application state without triggering re-registrations.
|
|
35
|
+
|
|
28
36
|
### Tools
|
|
29
37
|
|
|
30
38
|
Tools are functions that the assistant can use to interact with your application. They can be provided through various mechanisms:
|
|
@@ -58,7 +58,7 @@ function MyRuntimeProvider({ children }: { children: React.ReactNode }) {
|
|
|
58
58
|
</Callout>
|
|
59
59
|
|
|
60
60
|
```tsx
|
|
61
|
-
import {
|
|
61
|
+
import { useAssistantInteractable, useInteractableState } from "@assistant-ui/react";
|
|
62
62
|
import { z } from "zod";
|
|
63
63
|
|
|
64
64
|
const taskBoardSchema = z.object({
|
|
@@ -74,11 +74,12 @@ const taskBoardSchema = z.object({
|
|
|
74
74
|
const taskBoardInitialState = { tasks: [] };
|
|
75
75
|
|
|
76
76
|
function TaskBoard() {
|
|
77
|
-
const
|
|
77
|
+
const id = useAssistantInteractable("taskBoard", {
|
|
78
78
|
description: "A task board showing the user's tasks",
|
|
79
79
|
stateSchema: taskBoardSchema,
|
|
80
80
|
initialState: taskBoardInitialState,
|
|
81
81
|
});
|
|
82
|
+
const [state, { setState }] = useInteractableState(id, taskBoardInitialState);
|
|
82
83
|
|
|
83
84
|
return (
|
|
84
85
|
<div>
|
|
@@ -147,7 +148,7 @@ This is especially useful for large state objects where regenerating the entire
|
|
|
147
148
|
You can render multiple interactables with the same `name` but different `id`s. Each gets its own update tool:
|
|
148
149
|
|
|
149
150
|
```tsx
|
|
150
|
-
import {
|
|
151
|
+
import { useAssistantInteractable, useInteractableState } from "@assistant-ui/react";
|
|
151
152
|
import { z } from "zod";
|
|
152
153
|
|
|
153
154
|
const noteSchema = z.object({
|
|
@@ -159,12 +160,13 @@ const noteSchema = z.object({
|
|
|
159
160
|
const noteInitialState = { title: "New Note", content: "", color: "yellow" as const };
|
|
160
161
|
|
|
161
162
|
function NoteCard({ noteId }: { noteId: string }) {
|
|
162
|
-
|
|
163
|
+
useAssistantInteractable("note", {
|
|
163
164
|
id: noteId,
|
|
164
165
|
description: "A sticky note",
|
|
165
166
|
stateSchema: noteSchema,
|
|
166
167
|
initialState: noteInitialState,
|
|
167
168
|
});
|
|
169
|
+
const [state] = useInteractableState(noteId, noteInitialState);
|
|
168
170
|
|
|
169
171
|
return <div>{state.title}</div>;
|
|
170
172
|
}
|
|
@@ -187,12 +189,13 @@ When multiple interactables are present, you can mark one as "selected" to tell
|
|
|
187
189
|
|
|
188
190
|
```tsx
|
|
189
191
|
function NoteCard({ noteId }: { noteId: string }) {
|
|
190
|
-
|
|
192
|
+
useAssistantInteractable("note", {
|
|
191
193
|
id: noteId,
|
|
192
194
|
description: "A sticky note",
|
|
193
195
|
stateSchema: noteSchema,
|
|
194
196
|
initialState: noteInitialState,
|
|
195
197
|
});
|
|
198
|
+
const [state, { setSelected }] = useInteractableState(noteId, noteInitialState);
|
|
196
199
|
|
|
197
200
|
return (
|
|
198
201
|
<div onClick={() => setSelected(true)}>
|
|
@@ -206,12 +209,12 @@ The AI sees `(SELECTED)` in the system prompt for the focused interactable, allo
|
|
|
206
209
|
|
|
207
210
|
## API Reference
|
|
208
211
|
|
|
209
|
-
### `
|
|
212
|
+
### `useAssistantInteractable`
|
|
210
213
|
|
|
211
|
-
|
|
214
|
+
Registers an interactable with the AI assistant. Returns the instance id.
|
|
212
215
|
|
|
213
216
|
```tsx
|
|
214
|
-
const
|
|
217
|
+
const id = useAssistantInteractable(name, config);
|
|
215
218
|
```
|
|
216
219
|
|
|
217
220
|
**Parameters:**
|
|
@@ -221,43 +224,37 @@ const [state, setState, meta] = useInteractable<TState>(name, config);
|
|
|
221
224
|
| `name` | `string` | Name for the interactable (used in tool names) |
|
|
222
225
|
| `config.description` | `string` | Description shown to the AI |
|
|
223
226
|
| `config.stateSchema` | `StandardSchemaV1 \| JSONSchema7` | Schema for the state (e.g., a Zod schema) |
|
|
224
|
-
| `config.initialState` | `
|
|
227
|
+
| `config.initialState` | `unknown` | Initial state value |
|
|
225
228
|
| `config.id` | `string?` | Optional unique instance ID (auto-generated if omitted) |
|
|
226
229
|
| `config.selected` | `boolean?` | Whether this interactable is selected |
|
|
227
230
|
|
|
228
|
-
**Returns:** `
|
|
231
|
+
**Returns:** `string` — the instance id (auto-generated or provided).
|
|
229
232
|
|
|
230
|
-
|
|
231
|
-
| --- | --- | --- |
|
|
232
|
-
| `state` | `TState` | Current state |
|
|
233
|
-
| `setState` | `(updater: TState \| (prev: TState) => TState) => void` | State setter (like `useState`) |
|
|
234
|
-
| `meta.id` | `string` | The instance ID (auto-generated or provided) |
|
|
235
|
-
| `meta.setSelected` | `(selected: boolean) => void` | Mark this interactable as selected |
|
|
236
|
-
|
|
237
|
-
### `makeInteractable`
|
|
233
|
+
### `useInteractableState`
|
|
238
234
|
|
|
239
|
-
|
|
235
|
+
Reads and writes the state of a registered interactable.
|
|
240
236
|
|
|
241
237
|
```tsx
|
|
242
|
-
|
|
238
|
+
const [state, { setState, setSelected, isPending, error, flush }] = useInteractableState<TState>(id, fallback?);
|
|
239
|
+
```
|
|
243
240
|
|
|
244
|
-
|
|
245
|
-
name: "taskBoard",
|
|
246
|
-
description: "A task board showing the user's tasks",
|
|
247
|
-
stateSchema: taskBoardSchema,
|
|
248
|
-
initialState: taskBoardInitialState,
|
|
249
|
-
});
|
|
241
|
+
**Parameters:**
|
|
250
242
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
243
|
+
| Parameter | Type | Description |
|
|
244
|
+
| --- | --- | --- |
|
|
245
|
+
| `id` | `string` | The interactable instance id (from `useAssistantInteractable`) |
|
|
246
|
+
| `fallback` | `TState?` | Fallback value before the interactable is registered |
|
|
247
|
+
|
|
248
|
+
**Returns:** `[state, methods]`
|
|
249
|
+
|
|
250
|
+
| Return | Type | Description |
|
|
251
|
+
| --- | --- | --- |
|
|
252
|
+
| `state` | `TState` | Current state |
|
|
253
|
+
| `setState` | `(updater: TState \| (prev: TState) => TState) => void` | State setter (like `useState`) |
|
|
254
|
+
| `setSelected` | `(selected: boolean) => void` | Mark this interactable as selected |
|
|
255
|
+
| `isPending` | `boolean` | Whether a persistence save is in-flight |
|
|
256
|
+
| `error` | `unknown` | Error from the last failed save |
|
|
257
|
+
| `flush` | `() => Promise<void>` | Force an immediate persistence save |
|
|
261
258
|
|
|
262
259
|
### `Interactables`
|
|
263
260
|
|
|
@@ -271,7 +268,7 @@ const aui = useAui({
|
|
|
271
268
|
|
|
272
269
|
## How It Works
|
|
273
270
|
|
|
274
|
-
When you call `
|
|
271
|
+
When you call `useAssistantInteractable("taskBoard", config)`:
|
|
275
272
|
|
|
276
273
|
1. **Registration** — the interactable is registered in the `interactables` scope with its name, description, schema, and initial state.
|
|
277
274
|
2. **Tool generation** — an `update_taskBoard` frontend tool is automatically created with a partial schema (all fields optional). For multiple instances, tools are named `update_{name}_{id}`.
|
|
@@ -280,6 +277,62 @@ When you call `useInteractable("taskBoard", config)`:
|
|
|
280
277
|
5. **Partial merge** — only the fields the AI sends are updated; the rest are preserved.
|
|
281
278
|
6. **Bidirectional updates** — when the AI calls the tool, the state updates and React re-renders. When the user updates state via `setState`, the model context is notified so the AI sees the latest state on the next turn.
|
|
282
279
|
|
|
280
|
+
## Persistence
|
|
281
|
+
|
|
282
|
+
By default, interactable state is in-memory and lost on page refresh. You can add persistence by providing a save callback:
|
|
283
|
+
|
|
284
|
+
```tsx
|
|
285
|
+
import { useEffect } from "react";
|
|
286
|
+
import { useAui, Interactables } from "@assistant-ui/react";
|
|
287
|
+
|
|
288
|
+
function MyRuntimeProvider({ children }) {
|
|
289
|
+
const aui = useAui({ interactables: Interactables() });
|
|
290
|
+
|
|
291
|
+
useEffect(() => {
|
|
292
|
+
// Set up persistence adapter
|
|
293
|
+
aui.interactables().setPersistenceAdapter({
|
|
294
|
+
save: async (state) => {
|
|
295
|
+
localStorage.setItem("interactables", JSON.stringify(state));
|
|
296
|
+
},
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
// Restore saved state on mount
|
|
300
|
+
const saved = localStorage.getItem("interactables");
|
|
301
|
+
if (saved) {
|
|
302
|
+
aui.interactables().importState(JSON.parse(saved));
|
|
303
|
+
}
|
|
304
|
+
}, [aui]);
|
|
305
|
+
|
|
306
|
+
return /* ... */;
|
|
307
|
+
}
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
### Sync Status
|
|
311
|
+
|
|
312
|
+
When a persistence adapter is set, `useInteractableState` exposes sync metadata:
|
|
313
|
+
|
|
314
|
+
```tsx
|
|
315
|
+
const [state, { setState, isPending, error, flush }] = useInteractableState(id, fallback);
|
|
316
|
+
|
|
317
|
+
// isPending — true while a save is in-flight
|
|
318
|
+
// error — the error from the last failed save, if any
|
|
319
|
+
// flush() — force an immediate save (useful before navigation)
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
State changes are automatically debounced (500ms) before saving. When a component unregisters, any pending save is flushed immediately.
|
|
323
|
+
|
|
324
|
+
### Export / Import
|
|
325
|
+
|
|
326
|
+
For custom persistence strategies, use `exportState` and `importState` directly:
|
|
327
|
+
|
|
328
|
+
```tsx
|
|
329
|
+
const snapshot = aui.interactables().exportState();
|
|
330
|
+
// => { "note-1": { name: "note", state: { title: "Hello" } }, ... }
|
|
331
|
+
|
|
332
|
+
aui.interactables().importState(snapshot);
|
|
333
|
+
// Imported state is picked up when components next register
|
|
334
|
+
```
|
|
335
|
+
|
|
283
336
|
## Combining with Tools
|
|
284
337
|
|
|
285
338
|
You can use `Interactables` alongside `Tools`:
|
|
@@ -290,3 +343,12 @@ const aui = useAui({
|
|
|
290
343
|
interactables: Interactables(),
|
|
291
344
|
});
|
|
292
345
|
```
|
|
346
|
+
|
|
347
|
+
## Full Example
|
|
348
|
+
|
|
349
|
+
See the complete [with-interactables example](https://github.com/assistant-ui/assistant-ui/tree/main/examples/with-interactables) for a working implementation featuring:
|
|
350
|
+
|
|
351
|
+
- **Task Board** — single-instance interactable with a custom `manage_tasks` tool
|
|
352
|
+
- **Sticky Notes** — multi-instance interactables with selection and partial updates
|
|
353
|
+
- **localStorage persistence** — state survives page refresh via `setPersistenceAdapter`
|
|
354
|
+
- **Sync indicator** — spinning icon while a save is in-flight (`isPending`)
|
|
@@ -748,6 +748,35 @@ useAssistantToolUI({
|
|
|
748
748
|
server.
|
|
749
749
|
</Callout>
|
|
750
750
|
|
|
751
|
+
## Per-Property Streaming Status
|
|
752
|
+
|
|
753
|
+
When rendering a tool UI, you can track which arguments have finished streaming using `useToolArgsStatus`. This must be used inside a tool-call message part context.
|
|
754
|
+
|
|
755
|
+
```tsx
|
|
756
|
+
import { useToolArgsStatus } from "@assistant-ui/react";
|
|
757
|
+
|
|
758
|
+
const WeatherUI = makeAssistantToolUI({
|
|
759
|
+
toolName: "weather",
|
|
760
|
+
render: ({ args }) => {
|
|
761
|
+
const { status, propStatus } = useToolArgsStatus<{
|
|
762
|
+
location: string;
|
|
763
|
+
unit: string;
|
|
764
|
+
}>();
|
|
765
|
+
|
|
766
|
+
return (
|
|
767
|
+
<div>
|
|
768
|
+
<span className={propStatus.location === "streaming" ? "animate-pulse" : ""}>
|
|
769
|
+
{args.location ?? "..."}
|
|
770
|
+
</span>
|
|
771
|
+
{status === "complete" && <WeatherChart data={args} />}
|
|
772
|
+
</div>
|
|
773
|
+
);
|
|
774
|
+
},
|
|
775
|
+
});
|
|
776
|
+
```
|
|
777
|
+
|
|
778
|
+
`propStatus` maps each key to `"streaming"` | `"complete"` once the key appears in the partial JSON. Keys not yet present in the stream are absent from `propStatus`.
|
|
779
|
+
|
|
751
780
|
## Related Guides
|
|
752
781
|
|
|
753
782
|
- [Tools Guide](/docs/guides/tools) - Learn how to create and use tools with AI models
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Realtime Voice
|
|
3
|
+
description: Bidirectional realtime voice conversations with AI agents.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
import { VoiceSample } from "@/components/docs/samples/voice";
|
|
7
|
+
|
|
8
|
+
assistant-ui supports realtime bidirectional voice via the `RealtimeVoiceAdapter` interface. This enables live voice conversations where the user speaks into their microphone and the AI agent responds with audio, with transcripts appearing in the thread in real time.
|
|
9
|
+
|
|
10
|
+
<VoiceSample />
|
|
11
|
+
|
|
12
|
+
## How It Works
|
|
13
|
+
|
|
14
|
+
Unlike [Speech Synthesis](/docs/guides/speech) (text-to-speech) and [Dictation](/docs/guides/dictation) (speech-to-text), the voice adapter handles **both directions simultaneously** — the user's microphone audio is streamed to the agent, and the agent's audio response is played back, all while transcripts are appended to the message thread.
|
|
15
|
+
|
|
16
|
+
| Feature | Adapter | Direction |
|
|
17
|
+
|---------|---------|-----------|
|
|
18
|
+
| [Speech Synthesis](/docs/guides/speech) | `SpeechSynthesisAdapter` | Text → Audio (one message at a time) |
|
|
19
|
+
| [Dictation](/docs/guides/dictation) | `DictationAdapter` | Audio → Text (into composer) |
|
|
20
|
+
| **Realtime Voice** | `RealtimeVoiceAdapter` | Audio ↔ Audio (bidirectional, live) |
|
|
21
|
+
|
|
22
|
+
## Configuration
|
|
23
|
+
|
|
24
|
+
Pass a `RealtimeVoiceAdapter` implementation to the runtime via `adapters.voice`:
|
|
25
|
+
|
|
26
|
+
```tsx
|
|
27
|
+
const runtime = useChatRuntime({
|
|
28
|
+
adapters: {
|
|
29
|
+
voice: new MyVoiceAdapter({ /* ... */ }),
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
When a voice adapter is provided, `capabilities.voice` is automatically set to `true`.
|
|
35
|
+
|
|
36
|
+
## Hooks
|
|
37
|
+
|
|
38
|
+
### useVoiceState
|
|
39
|
+
|
|
40
|
+
Returns the current voice session state, or `undefined` when no session is active.
|
|
41
|
+
|
|
42
|
+
```tsx
|
|
43
|
+
import { useVoiceState, useVoiceVolume } from "@assistant-ui/react";
|
|
44
|
+
|
|
45
|
+
const voiceState = useVoiceState();
|
|
46
|
+
// voiceState?.status.type — "starting" | "running" | "ended"
|
|
47
|
+
// voiceState?.isMuted — boolean
|
|
48
|
+
// voiceState?.mode — "listening" | "speaking"
|
|
49
|
+
|
|
50
|
+
const volume = useVoiceVolume();
|
|
51
|
+
// volume — number (0–1, real-time audio level via separate subscription)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### useVoiceControls
|
|
55
|
+
|
|
56
|
+
Returns methods to control the voice session.
|
|
57
|
+
|
|
58
|
+
```tsx
|
|
59
|
+
import { useVoiceControls } from "@assistant-ui/react";
|
|
60
|
+
|
|
61
|
+
const { connect, disconnect, mute, unmute } = useVoiceControls();
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## UI Example
|
|
65
|
+
|
|
66
|
+
```tsx
|
|
67
|
+
import { useVoiceState, useVoiceControls } from "@assistant-ui/react";
|
|
68
|
+
import { PhoneIcon, PhoneOffIcon, MicIcon, MicOffIcon } from "lucide-react";
|
|
69
|
+
|
|
70
|
+
function VoiceControls() {
|
|
71
|
+
const voiceState = useVoiceState();
|
|
72
|
+
const { connect, disconnect, mute, unmute } = useVoiceControls();
|
|
73
|
+
|
|
74
|
+
const isRunning = voiceState?.status.type === "running";
|
|
75
|
+
const isStarting = voiceState?.status.type === "starting";
|
|
76
|
+
const isMuted = voiceState?.isMuted ?? false;
|
|
77
|
+
|
|
78
|
+
if (!isRunning && !isStarting) {
|
|
79
|
+
return (
|
|
80
|
+
<button onClick={() => connect()}>
|
|
81
|
+
<PhoneIcon /> Connect
|
|
82
|
+
</button>
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return (
|
|
87
|
+
<>
|
|
88
|
+
<button onClick={() => (isMuted ? unmute() : mute())} disabled={!isRunning}>
|
|
89
|
+
{isMuted ? <MicOffIcon /> : <MicIcon />}
|
|
90
|
+
{isMuted ? "Unmute" : "Mute"}
|
|
91
|
+
</button>
|
|
92
|
+
<button onClick={() => disconnect()}>
|
|
93
|
+
<PhoneOffIcon /> Disconnect
|
|
94
|
+
</button>
|
|
95
|
+
</>
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Custom Adapters
|
|
101
|
+
|
|
102
|
+
Implement the `RealtimeVoiceAdapter` interface to integrate with any voice provider.
|
|
103
|
+
|
|
104
|
+
### RealtimeVoiceAdapter Interface
|
|
105
|
+
|
|
106
|
+
```tsx
|
|
107
|
+
import type { RealtimeVoiceAdapter } from "@assistant-ui/react";
|
|
108
|
+
|
|
109
|
+
class MyVoiceAdapter implements RealtimeVoiceAdapter {
|
|
110
|
+
connect(options: {
|
|
111
|
+
abortSignal?: AbortSignal;
|
|
112
|
+
}): RealtimeVoiceAdapter.Session {
|
|
113
|
+
// Establish connection to your voice service
|
|
114
|
+
return {
|
|
115
|
+
get status() { /* ... */ },
|
|
116
|
+
get isMuted() { /* ... */ },
|
|
117
|
+
|
|
118
|
+
disconnect: () => { /* ... */ },
|
|
119
|
+
mute: () => { /* ... */ },
|
|
120
|
+
unmute: () => { /* ... */ },
|
|
121
|
+
|
|
122
|
+
onStatusChange: (callback) => {
|
|
123
|
+
// Status: { type: "starting" } → { type: "running" } → { type: "ended", reason }
|
|
124
|
+
return () => {}; // Return unsubscribe
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
onTranscript: (callback) => {
|
|
128
|
+
// callback({ role: "user" | "assistant", text: "...", isFinal: true })
|
|
129
|
+
// Transcripts are automatically appended as messages in the thread.
|
|
130
|
+
return () => {};
|
|
131
|
+
},
|
|
132
|
+
|
|
133
|
+
// Report who is speaking (drives the VoiceOrb speaking animation)
|
|
134
|
+
onModeChange: (callback) => {
|
|
135
|
+
// callback("listening") — user's turn
|
|
136
|
+
// callback("speaking") — agent's turn
|
|
137
|
+
return () => {};
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
// Report real-time audio level (0–1) for visual feedback
|
|
141
|
+
onVolumeChange: (callback) => {
|
|
142
|
+
// callback(0.72) — drives VoiceOrb amplitude and waveform bar heights
|
|
143
|
+
return () => {};
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Session Lifecycle
|
|
151
|
+
|
|
152
|
+
The session status follows the same pattern as other adapters:
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
starting → running → ended
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
The `ended` status includes a `reason`:
|
|
159
|
+
- `"finished"` — session ended normally
|
|
160
|
+
- `"cancelled"` — session was cancelled by the user
|
|
161
|
+
- `"error"` — session ended due to an error (includes `error` field)
|
|
162
|
+
|
|
163
|
+
### Mode and Volume
|
|
164
|
+
|
|
165
|
+
All adapters must implement `onModeChange` and `onVolumeChange`. If your provider doesn't support these, return a no-op unsubscribe:
|
|
166
|
+
|
|
167
|
+
- **`onModeChange`** — Reports `"listening"` (user's turn) or `"speaking"` (agent's turn). The `VoiceOrb` switches to the active speaking animation.
|
|
168
|
+
- **`onVolumeChange`** — Reports a real-time audio level (`0`–`1`). The `VoiceOrb` modulates its amplitude and glow, and waveform bars scale to match.
|
|
169
|
+
|
|
170
|
+
When using `createVoiceSession`, these are handled automatically — call `session.emitMode()` and `session.emitVolume()` when your provider delivers data.
|
|
171
|
+
|
|
172
|
+
### Transcript Handling
|
|
173
|
+
|
|
174
|
+
Transcripts emitted via `onTranscript` are automatically appended to the message thread:
|
|
175
|
+
|
|
176
|
+
- **User transcripts** (`role: "user"`, `isFinal: true`) are appended as user messages.
|
|
177
|
+
- **Assistant transcripts** (`role: "assistant"`) are streamed into an assistant message. The message shows a "running" status until `isFinal: true` is received.
|
|
178
|
+
|
|
179
|
+
## Example: ElevenLabs Conversational AI
|
|
180
|
+
|
|
181
|
+
[ElevenLabs Conversational AI](https://elevenlabs.io/docs/agents-platform/overview) provides realtime voice agents via WebRTC.
|
|
182
|
+
|
|
183
|
+
### Install Dependencies
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
npm install @elevenlabs/client
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Adapter
|
|
190
|
+
|
|
191
|
+
```tsx title="lib/elevenlabs-voice-adapter.ts"
|
|
192
|
+
import type { RealtimeVoiceAdapter, Unsubscribe } from "@assistant-ui/react";
|
|
193
|
+
import { VoiceConversation } from "@elevenlabs/client";
|
|
194
|
+
|
|
195
|
+
export class ElevenLabsVoiceAdapter implements RealtimeVoiceAdapter {
|
|
196
|
+
private _agentId: string;
|
|
197
|
+
|
|
198
|
+
constructor(options: { agentId: string }) {
|
|
199
|
+
this._agentId = options.agentId;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
connect(options: {
|
|
203
|
+
abortSignal?: AbortSignal;
|
|
204
|
+
}): RealtimeVoiceAdapter.Session {
|
|
205
|
+
const statusCallbacks = new Set<(s: RealtimeVoiceAdapter.Status) => void>();
|
|
206
|
+
const transcriptCallbacks = new Set<(t: RealtimeVoiceAdapter.TranscriptItem) => void>();
|
|
207
|
+
const modeCallbacks = new Set<(m: RealtimeVoiceAdapter.Mode) => void>();
|
|
208
|
+
const volumeCallbacks = new Set<(v: number) => void>();
|
|
209
|
+
|
|
210
|
+
let currentStatus: RealtimeVoiceAdapter.Status = { type: "starting" };
|
|
211
|
+
let isMuted = false;
|
|
212
|
+
let conversation: VoiceConversation | null = null;
|
|
213
|
+
let disposed = false;
|
|
214
|
+
|
|
215
|
+
const updateStatus = (status: RealtimeVoiceAdapter.Status) => {
|
|
216
|
+
if (disposed) return;
|
|
217
|
+
currentStatus = status;
|
|
218
|
+
for (const cb of statusCallbacks) cb(status);
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
const cleanup = () => {
|
|
222
|
+
disposed = true;
|
|
223
|
+
conversation = null;
|
|
224
|
+
statusCallbacks.clear();
|
|
225
|
+
transcriptCallbacks.clear();
|
|
226
|
+
modeCallbacks.clear();
|
|
227
|
+
volumeCallbacks.clear();
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const session: RealtimeVoiceAdapter.Session = {
|
|
231
|
+
get status() { return currentStatus; },
|
|
232
|
+
get isMuted() { return isMuted; },
|
|
233
|
+
disconnect: () => { conversation?.endSession(); cleanup(); },
|
|
234
|
+
mute: () => { conversation?.setMicMuted(true); isMuted = true; },
|
|
235
|
+
unmute: () => { conversation?.setMicMuted(false); isMuted = false; },
|
|
236
|
+
onStatusChange: (cb): Unsubscribe => {
|
|
237
|
+
statusCallbacks.add(cb);
|
|
238
|
+
return () => statusCallbacks.delete(cb);
|
|
239
|
+
},
|
|
240
|
+
onTranscript: (cb): Unsubscribe => {
|
|
241
|
+
transcriptCallbacks.add(cb);
|
|
242
|
+
return () => transcriptCallbacks.delete(cb);
|
|
243
|
+
},
|
|
244
|
+
onModeChange: (cb): Unsubscribe => {
|
|
245
|
+
modeCallbacks.add(cb);
|
|
246
|
+
return () => modeCallbacks.delete(cb);
|
|
247
|
+
},
|
|
248
|
+
onVolumeChange: (cb): Unsubscribe => {
|
|
249
|
+
volumeCallbacks.add(cb);
|
|
250
|
+
return () => volumeCallbacks.delete(cb);
|
|
251
|
+
},
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
if (options.abortSignal) {
|
|
255
|
+
options.abortSignal.addEventListener("abort", () => {
|
|
256
|
+
conversation?.endSession(); cleanup();
|
|
257
|
+
}, { once: true });
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const doConnect = async () => {
|
|
261
|
+
if (disposed) return;
|
|
262
|
+
try {
|
|
263
|
+
conversation = await VoiceConversation.startSession({
|
|
264
|
+
agentId: this._agentId,
|
|
265
|
+
onConnect: () => updateStatus({ type: "running" }),
|
|
266
|
+
onDisconnect: () => { updateStatus({ type: "ended", reason: "finished" }); cleanup(); },
|
|
267
|
+
onError: (msg) => { updateStatus({ type: "ended", reason: "error", error: new Error(msg) }); cleanup(); },
|
|
268
|
+
onModeChange: ({ mode }) => {
|
|
269
|
+
if (disposed) return;
|
|
270
|
+
for (const cb of modeCallbacks) cb(mode === "speaking" ? "speaking" : "listening");
|
|
271
|
+
},
|
|
272
|
+
onMessage: (msg) => {
|
|
273
|
+
if (disposed) return;
|
|
274
|
+
for (const cb of transcriptCallbacks) {
|
|
275
|
+
cb({ role: msg.role === "user" ? "user" : "assistant", text: msg.message, isFinal: true });
|
|
276
|
+
}
|
|
277
|
+
},
|
|
278
|
+
});
|
|
279
|
+
} catch (error) {
|
|
280
|
+
updateStatus({ type: "ended", reason: "error", error }); cleanup();
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
|
|
284
|
+
doConnect();
|
|
285
|
+
return session;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Usage
|
|
291
|
+
|
|
292
|
+
```tsx
|
|
293
|
+
import { ElevenLabsVoiceAdapter } from "@/lib/elevenlabs-voice-adapter";
|
|
294
|
+
|
|
295
|
+
const runtime = useChatRuntime({
|
|
296
|
+
adapters: {
|
|
297
|
+
voice: new ElevenLabsVoiceAdapter({
|
|
298
|
+
agentId: process.env.NEXT_PUBLIC_ELEVENLABS_AGENT_ID!,
|
|
299
|
+
}),
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
## Example: LiveKit
|
|
305
|
+
|
|
306
|
+
[LiveKit](https://livekit.io/) provides realtime voice via WebRTC rooms with transcription support.
|
|
307
|
+
|
|
308
|
+
### Install Dependencies
|
|
309
|
+
|
|
310
|
+
```bash
|
|
311
|
+
npm install livekit-client
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
### Usage
|
|
315
|
+
|
|
316
|
+
```tsx
|
|
317
|
+
import { LiveKitVoiceAdapter } from "@/lib/livekit-voice-adapter";
|
|
318
|
+
|
|
319
|
+
const runtime = useChatRuntime({
|
|
320
|
+
adapters: {
|
|
321
|
+
voice: new LiveKitVoiceAdapter({
|
|
322
|
+
url: process.env.NEXT_PUBLIC_LIVEKIT_URL!,
|
|
323
|
+
token: async () => {
|
|
324
|
+
const res = await fetch("/api/livekit-token", { method: "POST" });
|
|
325
|
+
const { token } = await res.json();
|
|
326
|
+
return token;
|
|
327
|
+
},
|
|
328
|
+
}),
|
|
329
|
+
},
|
|
330
|
+
});
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
See the `examples/with-livekit` directory in the repository for a complete implementation including the adapter and token endpoint.
|
|
@@ -37,6 +37,29 @@ Custom data events that can be rendered as UI at their position in the message s
|
|
|
37
37
|
|
|
38
38
|
You can use either the explicit format `{ type: "data", name: "workflow", data: {...} }` or the shorthand `data-*` prefixed format `{ type: "data-workflow", data: {...} }`. The prefixed format is automatically converted to a `DataMessagePart` (stripping the `data-` prefix as the `name`). Unknown message part types that don't match any built-in type are silently skipped with a console warning.
|
|
39
39
|
|
|
40
|
+
#### Streaming Data Parts
|
|
41
|
+
|
|
42
|
+
Data parts can be sent from the server using `appendData()` on the stream controller:
|
|
43
|
+
|
|
44
|
+
```ts
|
|
45
|
+
controller.appendData({
|
|
46
|
+
type: "data",
|
|
47
|
+
name: "chart",
|
|
48
|
+
data: { labels: ["Q1", "Q2"], values: [10, 20] },
|
|
49
|
+
});
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Register a renderer with `makeAssistantDataUI` to display data parts:
|
|
53
|
+
|
|
54
|
+
```tsx
|
|
55
|
+
import { makeAssistantDataUI } from "@assistant-ui/react";
|
|
56
|
+
|
|
57
|
+
const ChartUI = makeAssistantDataUI({
|
|
58
|
+
name: "chart",
|
|
59
|
+
render: ({ data }) => <MyChart data={data} />,
|
|
60
|
+
});
|
|
61
|
+
```
|
|
62
|
+
|
|
40
63
|
## Anatomy
|
|
41
64
|
|
|
42
65
|
```tsx
|