ai 7.0.101 → 7.0.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/index.d.ts +82 -20
  3. package/dist/index.js +2010 -330
  4. package/dist/index.js.map +1 -1
  5. package/dist/internal/index.js +1 -1
  6. package/docs/03-agents/07-workflow-agent.mdx +1 -1
  7. package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
  8. package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
  9. package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
  10. package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
  11. package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
  12. package/package.json +12 -12
  13. package/src/generate-text/execute-tools-from-stream.ts +7 -0
  14. package/src/generate-text/stream-text.ts +3 -1
  15. package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
  16. package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
  17. package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
  18. package/src/realtime/browser-realtime-audio.ts +107 -10
  19. package/src/realtime/browser-realtime-live-websocket.ts +247 -0
  20. package/src/realtime/browser-realtime-transport.ts +235 -69
  21. package/src/realtime/browser-realtime-webrtc.ts +582 -0
  22. package/src/realtime/encode-realtime-frame.ts +33 -0
  23. package/src/realtime/index.ts +1 -0
  24. package/src/realtime/realtime-attempt.ts +45 -0
  25. package/src/realtime/realtime-command-tracker.ts +81 -0
  26. package/src/realtime/realtime-event-channel.ts +170 -0
  27. package/src/realtime/realtime-event-reducer.ts +3 -0
  28. package/src/realtime/realtime-session-state.ts +65 -0
  29. package/src/realtime/realtime-session.ts +768 -218
  30. package/src/realtime/realtime-types.ts +1 -1
  31. package/src/realtime/validate-realtime-setup.ts +37 -0
@@ -91,7 +91,7 @@ import {
91
91
  } from "@ai-sdk/provider-utils";
92
92
 
93
93
  // src/version.ts
94
- var VERSION = true ? "7.0.101" : "0.0.0-test";
94
+ var VERSION = true ? "7.0.102" : "0.0.0-test";
95
95
 
96
96
  // src/util/download/download.ts
97
97
  var download = async ({
@@ -198,7 +198,7 @@ Workflow functions can time out or be interrupted by network failures. `Workflow
198
198
  'use client';
199
199
 
200
200
  import { useChat } from '@ai-sdk/react';
201
- import { WorkflowChatTransport } from '@ai-sdk/workflow';
201
+ import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
202
202
  import { useMemo } from 'react';
203
203
 
204
204
  export default function Chat() {
@@ -7,11 +7,15 @@ description: Learn how to build realtime voice conversations with the AI SDK.
7
7
 
8
8
  <Note type="warning">Realtime is an experimental feature.</Note>
9
9
 
10
- The AI SDK provides realtime models for bidirectional audio and text
11
- conversations over WebSockets. Realtime sessions run in the browser and connect
10
+ This guide covers legacy token-based, turn-based realtime conversations over
11
+ WebSockets. These sessions run in the browser and connect
12
12
  directly to the provider using a short-lived token that you create on your
13
13
  server. You can also route the connection through [AI Gateway](/providers/ai-sdk-providers/ai-gateway#realtime).
14
14
 
15
+ For OpenAI Live's continuous JSON/PCM16 WSS relay runtime and application-handled
16
+ client delegation, see
17
+ [`experimental_useRealtime`](/docs/reference/ai-sdk-ui/use-realtime#continuous-conversations).
18
+
15
19
  The typical flow is:
16
20
 
17
21
  1. The browser calls your setup endpoint.
@@ -20,6 +24,13 @@ The typical flow is:
20
24
  1. The model streams audio, text, and tool calls back to the browser.
21
25
  1. Tool calls are handled by your application with `onToolCall`.
22
26
 
27
+ For continuous **OpenAI Live** conversations, use an application-owned WebSocket
28
+ relay or optional WebRTC via `api.session`. Live uses client delegation: your
29
+ application handles delegated work and submits context rather than using the
30
+ turn-based tool loop below. See the
31
+ [`experimental_useRealtime` reference](/docs/reference/ai-sdk-ui/use-realtime)
32
+ for SDP setup, server-owned permissions, capture ownership, and graceful close.
33
+
23
34
  ## Setup Endpoint
24
35
 
25
36
  Create a setup endpoint that returns a short-lived token for the realtime
@@ -94,18 +105,21 @@ Then use the matching Gateway realtime model in the browser:
94
105
  import { experimental_useRealtime } from '@ai-sdk/react';
95
106
  import { gateway } from 'ai';
96
107
 
108
+ const model = gateway.experimental_realtime('openai/gpt-realtime-2');
109
+ const sessionConfig = {
110
+ instructions: 'You are a helpful assistant. Be concise.',
111
+ inputAudioTranscription: {},
112
+ voice: 'alloy',
113
+ turnDetection: { type: 'server-vad' as const },
114
+ };
115
+
97
116
  export default function RealtimePage() {
98
117
  const realtime = experimental_useRealtime({
99
- model: gateway.experimental_realtime('openai/gpt-realtime-2'),
118
+ model,
100
119
  api: {
101
120
  token: '/api/realtime/setup',
102
121
  },
103
- sessionConfig: {
104
- instructions: 'You are a helpful assistant. Be concise.',
105
- inputAudioTranscription: {},
106
- voice: 'alloy',
107
- turnDetection: { type: 'server-vad' },
108
- },
122
+ sessionConfig,
109
123
  });
110
124
 
111
125
  // ...
@@ -134,18 +148,21 @@ microphone audio, play model audio, send text messages, and render messages.
134
148
  import { openai } from '@ai-sdk/openai';
135
149
  import { experimental_useRealtime } from '@ai-sdk/react';
136
150
 
151
+ const model = openai.experimental_realtime('gpt-realtime');
152
+ const sessionConfig = {
153
+ instructions: 'You are a helpful assistant. Be concise.',
154
+ inputAudioTranscription: {},
155
+ voice: 'alloy',
156
+ turnDetection: { type: 'server-vad' as const },
157
+ };
158
+
137
159
  export default function RealtimePage() {
138
160
  const realtime = experimental_useRealtime({
139
- model: openai.experimental_realtime('gpt-realtime'),
161
+ model,
140
162
  api: {
141
163
  token: '/api/realtime/setup',
142
164
  },
143
- sessionConfig: {
144
- instructions: 'You are a helpful assistant. Be concise.',
145
- inputAudioTranscription: {},
146
- voice: 'alloy',
147
- turnDetection: { type: 'server-vad' },
148
- },
165
+ sessionConfig,
149
166
  });
150
167
 
151
168
  return (
@@ -166,6 +183,10 @@ export default function RealtimePage() {
166
183
  }
167
184
  ```
168
185
 
186
+ Keep model and session configuration objects stable across renders. Use module
187
+ scope as above, or `useMemo` when configuration depends on props. Replacing either
188
+ object replaces the hook's session.
189
+
169
190
  ## Tool Calling
170
191
 
171
192
  Realtime tool execution is client-driven. The provider sends tool calls over the
@@ -204,13 +225,15 @@ export async function POST(request: Request) {
204
225
 
205
226
  ### Client Tool Handler
206
227
 
207
- ```tsx filename='app/realtime/page.tsx' highlight="10-24"
228
+ ```tsx filename='app/realtime/page.tsx' highlight="12-26"
208
229
  import { openai } from '@ai-sdk/openai';
209
230
  import { experimental_useRealtime } from '@ai-sdk/react';
210
231
 
232
+ const model = openai.experimental_realtime('gpt-realtime');
233
+
211
234
  export default function RealtimePage() {
212
235
  const realtime = experimental_useRealtime({
213
- model: openai.experimental_realtime('gpt-realtime'),
236
+ model,
214
237
  api: {
215
238
  token: '/api/realtime/setup',
216
239
  },
@@ -162,7 +162,7 @@ For chat apps built on the [Workflow SDK](/docs/agents/workflow-agent), `Workflo
162
162
 
163
163
  ```tsx
164
164
  import { useChat } from '@ai-sdk/react';
165
- import { WorkflowChatTransport } from '@ai-sdk/workflow';
165
+ import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
166
166
  import { useMemo } from 'react';
167
167
 
168
168
  export default function Chat() {