ai 7.0.101 → 7.0.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/index.d.ts +82 -20
- package/dist/index.js +2010 -330
- package/dist/index.js.map +1 -1
- package/dist/internal/index.js +1 -1
- package/docs/03-agents/07-workflow-agent.mdx +1 -1
- package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
- package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
- package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
- package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
- package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
- package/package.json +12 -12
- package/src/generate-text/execute-tools-from-stream.ts +7 -0
- package/src/generate-text/stream-text.ts +3 -1
- package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
- package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
- package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
- package/src/realtime/browser-realtime-audio.ts +107 -10
- package/src/realtime/browser-realtime-live-websocket.ts +247 -0
- package/src/realtime/browser-realtime-transport.ts +235 -69
- package/src/realtime/browser-realtime-webrtc.ts +582 -0
- package/src/realtime/encode-realtime-frame.ts +33 -0
- package/src/realtime/index.ts +1 -0
- package/src/realtime/realtime-attempt.ts +45 -0
- package/src/realtime/realtime-command-tracker.ts +81 -0
- package/src/realtime/realtime-event-channel.ts +170 -0
- package/src/realtime/realtime-event-reducer.ts +3 -0
- package/src/realtime/realtime-session-state.ts +65 -0
- package/src/realtime/realtime-session.ts +768 -218
- package/src/realtime/realtime-types.ts +1 -1
- package/src/realtime/validate-realtime-setup.ts +37 -0
package/dist/internal/index.js
CHANGED
|
@@ -198,7 +198,7 @@ Workflow functions can time out or be interrupted by network failures. `Workflow
|
|
|
198
198
|
'use client';
|
|
199
199
|
|
|
200
200
|
import { useChat } from '@ai-sdk/react';
|
|
201
|
-
import { WorkflowChatTransport } from '@ai-sdk/workflow';
|
|
201
|
+
import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
|
|
202
202
|
import { useMemo } from 'react';
|
|
203
203
|
|
|
204
204
|
export default function Chat() {
|
|
@@ -7,11 +7,15 @@ description: Learn how to build realtime voice conversations with the AI SDK.
|
|
|
7
7
|
|
|
8
8
|
<Note type="warning">Realtime is an experimental feature.</Note>
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
This guide covers legacy token-based, turn-based realtime conversations over
|
|
11
|
+
WebSockets. These sessions run in the browser and connect
|
|
12
12
|
directly to the provider using a short-lived token that you create on your
|
|
13
13
|
server. You can also route the connection through [AI Gateway](/providers/ai-sdk-providers/ai-gateway#realtime).
|
|
14
14
|
|
|
15
|
+
For OpenAI Live's continuous JSON/PCM16 WSS relay runtime and application-handled
|
|
16
|
+
client delegation, see
|
|
17
|
+
[`experimental_useRealtime`](/docs/reference/ai-sdk-ui/use-realtime#continuous-conversations).
|
|
18
|
+
|
|
15
19
|
The typical flow is:
|
|
16
20
|
|
|
17
21
|
1. The browser calls your setup endpoint.
|
|
@@ -20,6 +24,13 @@ The typical flow is:
|
|
|
20
24
|
1. The model streams audio, text, and tool calls back to the browser.
|
|
21
25
|
1. Tool calls are handled by your application with `onToolCall`.
|
|
22
26
|
|
|
27
|
+
For continuous **OpenAI Live** conversations, use an application-owned WebSocket
|
|
28
|
+
relay or optional WebRTC via `api.session`. Live uses client delegation: your
|
|
29
|
+
application handles delegated work and submits context rather than using the
|
|
30
|
+
turn-based tool loop below. See the
|
|
31
|
+
[`experimental_useRealtime` reference](/docs/reference/ai-sdk-ui/use-realtime)
|
|
32
|
+
for SDP setup, server-owned permissions, capture ownership, and graceful close.
|
|
33
|
+
|
|
23
34
|
## Setup Endpoint
|
|
24
35
|
|
|
25
36
|
Create a setup endpoint that returns a short-lived token for the realtime
|
|
@@ -94,18 +105,21 @@ Then use the matching Gateway realtime model in the browser:
|
|
|
94
105
|
import { experimental_useRealtime } from '@ai-sdk/react';
|
|
95
106
|
import { gateway } from 'ai';
|
|
96
107
|
|
|
108
|
+
const model = gateway.experimental_realtime('openai/gpt-realtime-2');
|
|
109
|
+
const sessionConfig = {
|
|
110
|
+
instructions: 'You are a helpful assistant. Be concise.',
|
|
111
|
+
inputAudioTranscription: {},
|
|
112
|
+
voice: 'alloy',
|
|
113
|
+
turnDetection: { type: 'server-vad' as const },
|
|
114
|
+
};
|
|
115
|
+
|
|
97
116
|
export default function RealtimePage() {
|
|
98
117
|
const realtime = experimental_useRealtime({
|
|
99
|
-
model
|
|
118
|
+
model,
|
|
100
119
|
api: {
|
|
101
120
|
token: '/api/realtime/setup',
|
|
102
121
|
},
|
|
103
|
-
sessionConfig
|
|
104
|
-
instructions: 'You are a helpful assistant. Be concise.',
|
|
105
|
-
inputAudioTranscription: {},
|
|
106
|
-
voice: 'alloy',
|
|
107
|
-
turnDetection: { type: 'server-vad' },
|
|
108
|
-
},
|
|
122
|
+
sessionConfig,
|
|
109
123
|
});
|
|
110
124
|
|
|
111
125
|
// ...
|
|
@@ -134,18 +148,21 @@ microphone audio, play model audio, send text messages, and render messages.
|
|
|
134
148
|
import { openai } from '@ai-sdk/openai';
|
|
135
149
|
import { experimental_useRealtime } from '@ai-sdk/react';
|
|
136
150
|
|
|
151
|
+
const model = openai.experimental_realtime('gpt-realtime');
|
|
152
|
+
const sessionConfig = {
|
|
153
|
+
instructions: 'You are a helpful assistant. Be concise.',
|
|
154
|
+
inputAudioTranscription: {},
|
|
155
|
+
voice: 'alloy',
|
|
156
|
+
turnDetection: { type: 'server-vad' as const },
|
|
157
|
+
};
|
|
158
|
+
|
|
137
159
|
export default function RealtimePage() {
|
|
138
160
|
const realtime = experimental_useRealtime({
|
|
139
|
-
model
|
|
161
|
+
model,
|
|
140
162
|
api: {
|
|
141
163
|
token: '/api/realtime/setup',
|
|
142
164
|
},
|
|
143
|
-
sessionConfig
|
|
144
|
-
instructions: 'You are a helpful assistant. Be concise.',
|
|
145
|
-
inputAudioTranscription: {},
|
|
146
|
-
voice: 'alloy',
|
|
147
|
-
turnDetection: { type: 'server-vad' },
|
|
148
|
-
},
|
|
165
|
+
sessionConfig,
|
|
149
166
|
});
|
|
150
167
|
|
|
151
168
|
return (
|
|
@@ -166,6 +183,10 @@ export default function RealtimePage() {
|
|
|
166
183
|
}
|
|
167
184
|
```
|
|
168
185
|
|
|
186
|
+
Keep model and session configuration objects stable across renders. Use module
|
|
187
|
+
scope as above, or `useMemo` when configuration depends on props. Replacing either
|
|
188
|
+
object replaces the hook's session.
|
|
189
|
+
|
|
169
190
|
## Tool Calling
|
|
170
191
|
|
|
171
192
|
Realtime tool execution is client-driven. The provider sends tool calls over the
|
|
@@ -204,13 +225,15 @@ export async function POST(request: Request) {
|
|
|
204
225
|
|
|
205
226
|
### Client Tool Handler
|
|
206
227
|
|
|
207
|
-
```tsx filename='app/realtime/page.tsx' highlight="
|
|
228
|
+
```tsx filename='app/realtime/page.tsx' highlight="12-26"
|
|
208
229
|
import { openai } from '@ai-sdk/openai';
|
|
209
230
|
import { experimental_useRealtime } from '@ai-sdk/react';
|
|
210
231
|
|
|
232
|
+
const model = openai.experimental_realtime('gpt-realtime');
|
|
233
|
+
|
|
211
234
|
export default function RealtimePage() {
|
|
212
235
|
const realtime = experimental_useRealtime({
|
|
213
|
-
model
|
|
236
|
+
model,
|
|
214
237
|
api: {
|
|
215
238
|
token: '/api/realtime/setup',
|
|
216
239
|
},
|
|
@@ -162,7 +162,7 @@ For chat apps built on the [Workflow SDK](/docs/agents/workflow-agent), `Workflo
|
|
|
162
162
|
|
|
163
163
|
```tsx
|
|
164
164
|
import { useChat } from '@ai-sdk/react';
|
|
165
|
-
import { WorkflowChatTransport } from '@ai-sdk/workflow';
|
|
165
|
+
import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
|
|
166
166
|
import { useMemo } from 'react';
|
|
167
167
|
|
|
168
168
|
export default function Chat() {
|