ai 7.0.162 → 7.0.164

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +78 -0
  2. package/dist/index.d.ts +166 -41
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +539 -112
  5. package/dist/index.js.map +1 -1
  6. package/dist/internal/index.d.ts +43 -10
  7. package/dist/internal/index.d.ts.map +1 -1
  8. package/dist/internal/index.js +209 -24
  9. package/dist/internal/index.js.map +1 -1
  10. package/dist/test/index.d.ts +3 -1
  11. package/dist/test/index.d.ts.map +1 -1
  12. package/dist/test/index.js +2 -1
  13. package/dist/test/index.js.map +1 -1
  14. package/docs/02-foundations/02-providers-and-models.mdx +2 -0
  15. package/docs/03-agents/04-loop-control.mdx +34 -0
  16. package/docs/03-agents/06-memory.mdx +42 -0
  17. package/docs/03-ai-sdk-core/05-generating-text.mdx +1 -0
  18. package/docs/03-ai-sdk-core/16-mcp-tools.mdx +12 -0
  19. package/docs/03-ai-sdk-core/36-realtime.mdx +74 -1
  20. package/docs/03-ai-sdk-core/36-transcription.mdx +51 -4
  21. package/docs/03-ai-sdk-core/40-middleware.mdx +7 -2
  22. package/docs/03-ai-sdk-core/60-telemetry.mdx +37 -4
  23. package/docs/04-ai-sdk-ui/50-stream-protocol.mdx +13 -0
  24. package/docs/06-advanced/04-caching.mdx +13 -4
  25. package/docs/07-reference/01-ai-sdk-core/01-generate-text.mdx +33 -28
  26. package/docs/07-reference/01-ai-sdk-core/02-stream-text.mdx +58 -1
  27. package/docs/07-reference/01-ai-sdk-core/11-transcribe.mdx +17 -2
  28. package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx +8 -1
  29. package/docs/07-reference/01-ai-sdk-core/31-ui-message.mdx +23 -0
  30. package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +37 -8
  31. package/docs/07-reference/04-ai-sdk-workflow/01-workflow-agent.mdx +1 -49
  32. package/docs/08-migration-guides/23-migration-guide-7-0.mdx +36 -34
  33. package/package.json +12 -12
  34. package/src/generate-object/generate-object.ts +2 -1
  35. package/src/generate-object/stream-object.ts +7 -3
  36. package/src/generate-text/generate-text.ts +23 -3
  37. package/src/generate-text/is-output-chunk.ts +1 -0
  38. package/src/generate-text/language-model-events.ts +3 -0
  39. package/src/generate-text/output.ts +90 -37
  40. package/src/generate-text/smooth-stream.ts +5 -0
  41. package/src/generate-text/steering.ts +5 -4
  42. package/src/generate-text/stream-language-model-call.ts +16 -1
  43. package/src/generate-text/stream-text-result.ts +29 -1
  44. package/src/generate-text/stream-text.ts +25 -1
  45. package/src/generate-text/to-response-messages.ts +6 -2
  46. package/src/generate-video/generate-video-result.ts +11 -2
  47. package/src/generate-video/generate-video.ts +26 -2
  48. package/src/model/as-transcription-model-v4.ts +25 -0
  49. package/src/prompt/convert-to-language-model-prompt.ts +86 -14
  50. package/src/realtime/index.ts +3 -0
  51. package/src/realtime/realtime-attempt.ts +1 -0
  52. package/src/realtime/realtime-session.ts +142 -21
  53. package/src/telemetry/create-telemetry-dispatcher.ts +8 -1
  54. package/src/telemetry/filter-telemetry-event.ts +165 -0
  55. package/src/telemetry/get-request-origin.ts +25 -0
  56. package/src/telemetry/tracing-channel-publisher.ts +19 -4
  57. package/src/test/mock-transcription-model-v4.ts +4 -0
  58. package/src/transcribe/transcribe-result.ts +20 -0
  59. package/src/transcribe/transcribe.ts +107 -20
  60. package/src/transcribe/transcription-events.ts +28 -8
  61. package/src/types/language-model-request-metadata.ts +3 -0
  62. package/src/types/realtime-model.ts +9 -0
  63. package/src/ui/convert-to-model-messages.ts +36 -28
  64. package/src/ui/index.ts +1 -0
  65. package/src/ui/process-ui-message-stream.ts +10 -0
  66. package/src/ui/ui-messages.ts +20 -0
  67. package/src/ui/validate-ui-messages.ts +21 -0
  68. package/src/ui-message-stream/to-ui-message-chunk.ts +72 -1
  69. package/src/ui-message-stream/ui-message-chunks.ts +31 -0
@@ -74,6 +74,13 @@ By default, both inputs and outputs are recorded. You can disable them by settin
74
74
  Disabling the recording of inputs and outputs can be useful for privacy, data transfer, and performance reasons.
75
75
  You might for example want to disable recording inputs if they contain sensitive information.
76
76
 
77
+ These settings filter content before lifecycle events reach telemetry integrations
78
+ and Node.js `ai:telemetry` subscribers. Raw provider bodies and provider metadata
79
+ are omitted when either recording option is disabled. Raw tracing span results
80
+ are also omitted in that case because they can contain both inputs and outputs.
81
+ Tracing span errors use a generic failure message.
82
+ Application lifecycle callbacks and returned results retain their full content.
83
+
77
84
  For `generateSpeech`, input telemetry contains the source text and output
78
85
  telemetry contains the generated audio byte length, media type, and format.
79
86
  For `transcribe` and `experimental_streamTranscribe`, telemetry contains the
@@ -102,6 +109,23 @@ such as `gen_ai.usage.characters`, `gen_ai.usage.seconds`, and
102
109
  callbacks, and can also be included as the supplemental `ai.response.usage`
103
110
  attribute by enabling `usage: true`.
104
111
 
112
+ ### Provider endpoints
113
+
114
+ For completed language-model calls, providers can report the known HTTP request
115
+ endpoint. `OpenTelemetry` records its host as `server.address` and its port as
116
+ `server.port` on the `chat` span. This includes default ports (443 for HTTPS and
117
+ 80 for HTTP) and works when input and output recording are disabled.
118
+
119
+ OpenAI Chat reports this information for both `generateText` and `streamText`,
120
+ including custom `baseURL` settings. Azure Chat and Bedrock Mantle Chat reuse
121
+ this adapter and also report their configured endpoint. Other providers leave these attributes
122
+ unset until they report their endpoint; no destination is inferred from the
123
+ provider or model name, including an unknown backend behind an intermediary.
124
+
125
+ The model-call end event's optional `request.url` contains only the HTTP origin.
126
+ Credentials, paths, queries, and fragments are removed before telemetry events
127
+ are emitted. The OpenTelemetry attributes contain only host and port.
128
+
105
129
  ### Opting out
106
130
 
107
131
  Telemetry is opt-out. To disable telemetry for a specific call, set `isEnabled: false`:
@@ -594,7 +618,7 @@ metrics. The `LegacyOpenTelemetry` integration does not emit this histogram.
594
618
 
595
619
  #### generateText / streamText
596
620
 
597
- For `generateText` and `streamText`, the integration records 3 types of spans:
621
+ For `generateText` and `streamText`, the integration records 4 types of spans:
598
622
 
599
623
  - **`invoke_agent {modelId}`** (default root span name, `INTERNAL`): covers the full operation including all steps and tool calls. Use [`agentSpanName`](#agent-span-names) to customize the name.
600
624
 
@@ -622,7 +646,16 @@ For `generateText` and `streamText`, the integration records 3 types of spans:
622
646
  - `gen_ai.usage.cache_creation.input_tokens`: cached input tokens created
623
647
  - `gen_ai.output.messages`: the output in [GenAI SemConv message format](#genai-message-format) (when `recordOutputs` is enabled)
624
648
 
625
- - **`chat {modelId}`** (step span, `CLIENT`): one span per LLM provider call, nested under the root span.
649
+ - **`step {n}`** (step span, `INTERNAL`): one span per step, nested under the root span. `n` is the step number, starting at 1. It contains the model call and tool spans of that step.
650
+
651
+ Initial attributes:
652
+ - `gen_ai.operation.name`: `"agent_step"`
653
+
654
+ `agent_step` is an AI SDK-defined custom operation name, not a well-known
655
+ OpenTelemetry GenAI operation. It marks an orchestration step, while the
656
+ nested `chat` span marks the model call.
657
+
658
+ - **`chat {modelId}`** (model call span, `CLIENT`): one span per LLM provider call, nested under the step span.
626
659
 
627
660
  Initial attributes:
628
661
  - `gen_ai.operation.name`: `"chat"`
@@ -655,11 +688,11 @@ For `generateText` and `streamText`, the integration records 3 types of spans:
655
688
  `streamText` with the `output` property instead.
656
689
  </Note>
657
690
 
658
- The deprecated object APIs emit the same span hierarchy as `generateText`/`streamText` with these additional attributes on the root span:
691
+ The deprecated object APIs emit the same spans as `generateText`/`streamText`, except that there is no `step {n}` span: the `chat {modelId}` span is nested directly under the root span. The root span has these additional attributes:
659
692
 
660
693
  - `gen_ai.output.type`: `"json"`
661
694
 
662
- The step spans also include `gen_ai.output.type: "json"`, and `gen_ai.output.messages` contains the generated object as a text part.
695
+ The `chat {modelId}` spans also include `gen_ai.output.type: "json"`, and `gen_ai.output.messages` contains the generated object as a text part.
663
696
 
664
697
  #### embed / embedMany
665
698
 
@@ -425,6 +425,19 @@ data: {"type":"tool-output-denied","toolCallId":"call_fJdQDqnXeGxTmr4E3YPSR7Ar"}
425
425
 
426
426
  ```
427
427
 
428
+ ### Steering Message Part
429
+
430
+ Experimental. A user message that was added to the run with [steering](/docs/agents/loop-control#steering). It is sent before the `start-step` part of the step that received it, and `useChat` adds it to the assistant message as a `steering-message` part.
431
+
432
+ Format: Server-Sent Event with JSON object
433
+
434
+ Example:
435
+
436
+ ```
437
+ data: {"type":"steering-message","id":"steer-1","parts":[{"type":"text","text":"I meant Paris, Texas."}]}
438
+
439
+ ```
440
+
428
441
  ### Start Step Part
429
442
 
430
443
  A part indicating the start of a step.
@@ -14,6 +14,7 @@ and the [`simulateReadableStream`](/docs/reference/ai-sdk-core/simulate-readable
14
14
 
15
15
  Language model middleware is a way to enhance the behavior of language models by intercepting and modifying the calls to the language model.
16
16
  Let's see how you can use language model middleware to cache responses.
17
+ Include the provider and model ID in the cache key alongside the call parameters to keep cached responses separate across models and providers.
17
18
 
18
19
  ```ts filename="ai/middleware.ts"
19
20
  import { Redis } from '@upstash/redis';
@@ -31,8 +32,12 @@ const redis = new Redis({
31
32
 
32
33
  export const cacheMiddleware: LanguageModelV4Middleware = {
33
34
  specificationVersion: 'v4',
34
- wrapGenerate: async ({ doGenerate, params }) => {
35
- const cacheKey = JSON.stringify(params);
35
+ wrapGenerate: async ({ doGenerate, params, model }) => {
36
+ const cacheKey = JSON.stringify({
37
+ provider: model.provider,
38
+ modelId: model.modelId,
39
+ params,
40
+ });
36
41
 
37
42
  const cached = (await redis.get(cacheKey)) as Awaited<
38
43
  ReturnType<LanguageModelV4['doGenerate']>
@@ -56,8 +61,12 @@ export const cacheMiddleware: LanguageModelV4Middleware = {
56
61
 
57
62
  return result;
58
63
  },
59
- wrapStream: async ({ doStream, params }) => {
60
- const cacheKey = JSON.stringify(params);
64
+ wrapStream: async ({ doStream, params, model }) => {
65
+ const cacheKey = JSON.stringify({
66
+ provider: model.provider,
67
+ modelId: model.modelId,
68
+ params,
69
+ });
61
70
 
62
71
  // Check if the result is in the cache
63
72
  const cached = await redis.get(cacheKey);
@@ -1277,6 +1277,11 @@ To see `generateText` in action, check out [these examples](#examples).
1277
1277
  {
1278
1278
  type: 'GenerateTextStepStartEvent',
1279
1279
  parameters: [
1280
+ {
1281
+ name: 'callId',
1282
+ type: 'string',
1283
+ description: 'Unique identifier for the generation call.',
1284
+ },
1280
1285
  {
1281
1286
  name: 'stepNumber',
1282
1287
  type: 'number',
@@ -1339,40 +1344,12 @@ To see `generateText` in action, check out [these examples](#examples).
1339
1344
  description:
1340
1345
  'Additional provider-specific options for this step.',
1341
1346
  },
1342
- {
1343
- name: 'timeout',
1344
- type: 'number | { totalMs?: number; stepMs?: number; chunkMs?: number } | undefined',
1345
- description:
1346
- 'Timeout configuration for the generation. Can be a number (milliseconds) or an object with totalMs, stepMs, chunkMs.',
1347
- },
1348
- {
1349
- name: 'headers',
1350
- type: 'Record<string, string | undefined> | undefined',
1351
- description: 'Additional HTTP headers sent with the request.',
1352
- },
1353
- {
1354
- name: 'stopWhen',
1355
- type: 'StopCondition<TOOLS> | Array<StopCondition<TOOLS>> | undefined',
1356
- description:
1357
- 'Condition(s) for stopping the generation. When the condition is an array, any of the conditions can be met to stop.',
1358
- },
1359
1347
  {
1360
1348
  name: 'output',
1361
1349
  type: 'OUTPUT | undefined',
1362
1350
  description:
1363
1351
  'The output specification for structured outputs, if configured.',
1364
1352
  },
1365
- {
1366
- name: 'abortSignal',
1367
- type: 'AbortSignal | undefined',
1368
- description: 'Abort signal for cancelling the operation.',
1369
- },
1370
- {
1371
- name: 'include',
1372
- type: '{ requestBody?: boolean; requestMessages?: boolean; responseBody?: boolean } | undefined',
1373
- description:
1374
- 'Settings for controlling what data is included in step results.',
1375
- },
1376
1353
  {
1377
1354
  name: 'runtimeContext',
1378
1355
  type: 'CONTEXT',
@@ -1930,6 +1907,13 @@ To see `generateText` in action, check out [these examples](#examples).
1930
1907
  {
1931
1908
  type: 'LanguageModelRequestMetadata',
1932
1909
  parameters: [
1910
+ {
1911
+ name: 'url',
1912
+ type: 'string',
1913
+ isOptional: true,
1914
+ description:
1915
+ 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
1916
+ },
1933
1917
  {
1934
1918
  name: 'messages',
1935
1919
  type: 'Array<ModelMessage>',
@@ -2386,6 +2370,13 @@ To see `generateText` in action, check out [these examples](#examples).
2386
2370
  {
2387
2371
  type: 'LanguageModelRequestMetadata',
2388
2372
  parameters: [
2373
+ {
2374
+ name: 'url',
2375
+ type: 'string',
2376
+ isOptional: true,
2377
+ description:
2378
+ 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
2379
+ },
2389
2380
  {
2390
2381
  name: 'messages',
2391
2382
  type: 'Array<ModelMessage>',
@@ -2811,6 +2802,13 @@ To see `generateText` in action, check out [these examples](#examples).
2811
2802
  {
2812
2803
  type: 'LanguageModelRequestMetadata',
2813
2804
  parameters: [
2805
+ {
2806
+ name: 'url',
2807
+ type: 'string',
2808
+ isOptional: true,
2809
+ description:
2810
+ 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
2811
+ },
2814
2812
  {
2815
2813
  name: 'messages',
2816
2814
  type: 'Array<ModelMessage>',
@@ -3192,6 +3190,13 @@ To see `generateText` in action, check out [these examples](#examples).
3192
3190
  {
3193
3191
  type: 'LanguageModelRequestMetadata',
3194
3192
  parameters: [
3193
+ {
3194
+ name: 'url',
3195
+ type: 'string',
3196
+ isOptional: true,
3197
+ description:
3198
+ 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
3199
+ },
3195
3200
  {
3196
3201
  name: 'messages',
3197
3202
  type: 'Array<ModelMessage>',
@@ -1032,7 +1032,7 @@ retain the content their contracts require.
1032
1032
  name: 'chunk',
1033
1033
  type: 'TextStreamPart<TOOLS>',
1034
1034
  description:
1035
- 'The stream part. This is the same union as `stream` and includes `start`, `start-step`, `text-start`, `text-delta`, `text-end`, `reasoning-start`, `reasoning-delta`, `reasoning-end`, `custom`, `source`, `file`, `reasoning-file`, `tool-call`, `tool-input-start`, `tool-input-delta`, `tool-input-end`, `tool-result`, `tool-error`, `tool-output-denied`, `tool-approval-request`, `tool-approval-response`, `finish-step`, `finish`, `abort`, `error`, and `raw`.',
1035
+ 'The stream part. This is the same union as `stream` and includes `start`, `steering-message`, `start-step`, `text-start`, `text-delta`, `text-end`, `reasoning-start`, `reasoning-delta`, `reasoning-end`, `custom`, `source`, `file`, `reasoning-file`, `tool-call`, `tool-input-start`, `tool-input-delta`, `tool-input-end`, `tool-result`, `tool-error`, `tool-output-denied`, `tool-approval-request`, `tool-approval-response`, `finish-step`, `finish`, `abort`, `error`, `raw`, and `ping`.',
1036
1036
  properties: [
1037
1037
  {
1038
1038
  type: 'TextStreamPart',
@@ -3017,6 +3017,12 @@ retain the content their contracts require.
3017
3017
  {
3018
3018
  type: 'LanguageModelRequestMetadata',
3019
3019
  parameters: [
3020
+ {
3021
+ name: 'url',
3022
+ type: 'string',
3023
+ isOptional: true,
3024
+ description: 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
3025
+ },
3020
3026
  {
3021
3027
  name: 'messages',
3022
3028
  type: 'Array<ModelMessage>',
@@ -3309,6 +3315,13 @@ retain the content their contracts require.
3309
3315
  {
3310
3316
  type: 'RequestMetadata',
3311
3317
  parameters: [
3318
+ {
3319
+ name: 'url',
3320
+ type: 'string',
3321
+ isOptional: true,
3322
+ description:
3323
+ 'The HTTP origin reported by the provider, without URL credentials, paths, queries or fragments.',
3324
+ },
3312
3325
  {
3313
3326
  name: 'messages',
3314
3327
  type: 'Array<ModelMessage>',
@@ -3418,6 +3431,17 @@ retain the content their contracts require.
3418
3431
  description:
3419
3432
  'A stream with all events, including text deltas, tool calls, tool results, and errors. Well-formed mid-stream provider failures are emitted in error parts as StreamProviderError instances. You can use it as either an AsyncIterable or a ReadableStream. Only errors that stop the stream, such as network errors, are thrown.',
3420
3433
  properties: [
3434
+ {
3435
+ type: 'TextStreamPingPart',
3436
+ parameters: [
3437
+ {
3438
+ name: 'type',
3439
+ type: "'ping'",
3440
+ description:
3441
+ 'A transient provider heartbeat. It contains no generated content, does not reset model-output timeouts, and is omitted from text and UI message streams. Providers may require an explicit opt-in.',
3442
+ },
3443
+ ],
3444
+ },
3421
3445
  {
3422
3446
  type: 'TextStreamTextEndPart',
3423
3447
  description: 'End of a text block, including optional citations supporting that block.',
@@ -3677,6 +3701,33 @@ retain the content their contracts require.
3677
3701
  },
3678
3702
  ],
3679
3703
  },
3704
+ {
3705
+ type: 'TextStreamPart',
3706
+ parameters: [
3707
+ {
3708
+ name: 'type',
3709
+ type: "'steering-message'",
3710
+ description:
3711
+ 'Experimental. A user message that was added to the run with `experimental_steering`. It is streamed before the `start-step` part of the step that received it.',
3712
+ },
3713
+ {
3714
+ name: 'id',
3715
+ type: 'string',
3716
+ description: 'The id of the steering message.',
3717
+ },
3718
+ {
3719
+ name: 'message',
3720
+ type: 'UserModelMessage',
3721
+ description: 'The user message that was added to the run.',
3722
+ },
3723
+ {
3724
+ name: 'stepNumber',
3725
+ type: 'number',
3726
+ description:
3727
+ 'The number of the step whose model call received the message.',
3728
+ },
3729
+ ],
3730
+ },
3680
3731
  {
3681
3732
  type: 'TextStreamPart',
3682
3733
  parameters: [
@@ -3694,6 +3745,12 @@ retain the content their contracts require.
3694
3745
  {
3695
3746
  type: 'LanguageModelRequestMetadata',
3696
3747
  parameters: [
3748
+ {
3749
+ name: 'url',
3750
+ type: 'string',
3751
+ isOptional: true,
3752
+ description: 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
3753
+ },
3697
3754
  {
3698
3755
  name: 'messages',
3699
3756
  type: 'Array<ModelMessage>',
@@ -38,7 +38,15 @@ console.log(transcript);
38
38
  {
39
39
  name: 'audio',
40
40
  type: 'DataContent (string | Uint8Array | ArrayBuffer | Buffer) | URL',
41
- description: 'The audio file to generate the transcript from.',
41
+ description:
42
+ 'The audio file to generate the transcript from. HTTP(S) URLs are sent directly to models with native URL support unless a custom download function is supplied; other models download the audio locally.',
43
+ },
44
+ {
45
+ name: 'timestampGranularities',
46
+ type: "Array<'word' | 'segment'>",
47
+ isOptional: true,
48
+ description:
49
+ 'Word and segment timings to request. Supported by the OpenAI, Azure Speech, Vertex Gemini, and xAI adapters. Explicit provider options take precedence; these adapters warn about unavailable requested timings. AI Gateway warns that it does not forward this option; use provider options or a direct provider model. Omit this option to keep provider defaults. An empty array requests no additional timing detail; it does not suppress timings returned by default.',
42
50
  },
43
51
  {
44
52
  name: 'providerOptions',
@@ -76,7 +84,7 @@ console.log(transcript);
76
84
  type: '(options: { url: URL; abortSignal?: AbortSignal }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>',
77
85
  isOptional: true,
78
86
  description:
79
- 'Custom download function for fetching audio from URLs. Use `createDownload()` from `ai` to create a download function with custom size limits, e.g. `createDownload({ maxBytes: 50 * 1024 * 1024 })`. Default: built-in download with 2 GiB limit.',
87
+ 'Custom download function for fetching audio from URLs. Supplying this option forces local downloading even for models with native URL support. Use `createDownload()` from `ai` to create a download function with custom size limits, e.g. `createDownload({ maxBytes: 50 * 1024 * 1024 })`. Default: native URL transcription when supported, otherwise built-in download with 2 GiB limit.',
80
88
  },
81
89
  ]}
82
90
  />
@@ -96,6 +104,13 @@ console.log(transcript);
96
104
  description:
97
105
  'An array of transcript segments, each containing a portion of the transcribed text along with its start and end times in seconds.',
98
106
  },
107
+ {
108
+ name: 'words',
109
+ type: 'Array<{ text: string; startSecond: number; endSecond: number }>',
110
+ isOptional: true,
111
+ description:
112
+ 'Word-level timings when the adapter exposes them, including timings returned by default. Undefined when unavailable. Existing segments behavior is preserved: some providers also put word timings in segments.',
113
+ },
99
114
  {
100
115
  name: 'language',
101
116
  type: 'string | undefined',
@@ -220,7 +220,8 @@ console.log(videos);
220
220
  {
221
221
  name: 'video',
222
222
  type: 'GeneratedFile',
223
- description: 'The first video that was generated.',
223
+ description:
224
+ 'The first video available to the SDK. For upload-only results, accessing this property throws NoVideoGeneratedError; use uploadedVideos instead.',
224
225
  properties: [
225
226
  {
226
227
  type: 'GeneratedFile',
@@ -273,6 +274,12 @@ console.log(videos);
273
274
  },
274
275
  ],
275
276
  },
277
+ {
278
+ name: 'uploadedVideos',
279
+ type: 'number',
280
+ description:
281
+ 'Number of videos delivered directly to caller-managed storage by the provider. These are not downloaded and are not included in videos.',
282
+ },
276
283
  {
277
284
  name: 'warnings',
278
285
  type: 'Warning[]',
@@ -320,6 +320,29 @@ type DataUIPart<DATA_TYPES extends UIDataTypes> = ValueOf<{
320
320
  }>;
321
321
  ```
322
322
 
323
+ ### `SteeringMessageUIPart`
324
+
325
+ <Note>
326
+ This part is experimental and exported as
327
+ `Experimental_SteeringMessageUIPart`.
328
+ </Note>
329
+
330
+ A user message that was added to the run with [steering](/docs/agents/loop-control#steering) while the assistant message was generated. It is placed before the `step-start` part of the step that received it.
331
+
332
+ ```typescript
333
+ type SteeringMessageUIPart = {
334
+ type: 'steering-message';
335
+ /**
336
+ * The id of the steering message.
337
+ */
338
+ id: string;
339
+ /**
340
+ * The content of the user message.
341
+ */
342
+ parts: Array<TextUIPart | FileUIPart>;
343
+ };
344
+ ```
345
+
323
346
  ### `StepStartUIPart`
324
347
 
325
348
  A step boundary part of a message.
@@ -13,7 +13,8 @@ Creates a browser-side realtime session for bidirectional audio and text
13
13
  conversations with a realtime provider model.
14
14
 
15
15
  The hook supports token-based provider WebSockets, application-owned WebSocket
16
- relays, and optional WebRTC where the model declares support. It provides controls for capture and playback, plus turn-based text input
16
+ relays, optional WebRTC, and provider-owned transports where the model declares
17
+ support. It provides controls for capture and playback, plus supported turn-based text input
17
18
  and tool output. Turn-based conversation messages use `UIMessage[]`; continuous
18
19
  transcript fragments remain in `session`.
19
20
 
@@ -184,12 +185,33 @@ updates remain unsupported.
184
185
 
185
186
  ## Lifecycle and failure handling
186
187
 
188
+ ### Provider-owned sessions
189
+
190
+ For [Mistral voice agents](/providers/ai-sdk-providers/mistral#realtime-voice-agents),
191
+ use `api: { provider: '/api/mistral-realtime' }`. The Mistral provider handles
192
+ LiveKit internally.
193
+ The hook calls `POST` on that endpoint to create a session and
194
+ `DELETE ?sessionId=...` to close it. The server uses
195
+ `mistral.experimental_realtime.createSession()` and `closeSession()` with its API
196
+ key. Keep the model, voice, instructions, and allowed settings on the server.
197
+
198
+ `connect()` starts microphone capture and remote audio playback. Mistral
199
+ transcripts populate `messages`; audio travels through the provider's LiveKit
200
+ transport. Await `close()` to finish server-side cleanup. `disconnect()` and
201
+ unmount release local media and initiate cleanup without waiting. Client tools
202
+ use `onToolCall` and `addToolOutput`. Text, manual audio-buffer, and response
203
+ commands are unsupported. See the provider guide for the complete route and
204
+ client example.
205
+
206
+ ### Shared lifecycle
207
+
187
208
  Use `status === 'connected'` as the readiness signal; `connect()` does not promise
188
209
  to wait for provider readiness. Operational startup errors are reported through
189
210
  `onError` and `status`; the legacy resolve-and-report behavior is preserved.
190
211
 
191
- `close()` stops local capture and submissions, then drains events until the provider
192
- confirms final usage or the close deadline expires. A failed close-command send
212
+ For continuous Live sessions, `close()` stops local capture and submissions, then
213
+ drains events until the provider confirms final usage or the close deadline
214
+ expires. A failed close-command send
193
215
  uses the shorter accepted-event drain rather than waiting for an acknowledgment
194
216
  to an unsent command. Read `session.finalization`: a fulfilled close promise does
195
217
  not by itself confirm usage. `disconnect()` and unmount release resources immediately.
@@ -244,9 +266,9 @@ and completion of delegated work do not prove that the corresponding audio was h
244
266
  },
245
267
  {
246
268
  name: 'api',
247
- type: '{ token: string } | { websocket: string; protocols?: string[] } | { session: string }',
269
+ type: '{ token: string } | { websocket: string; protocols?: string[] } | { session: string } | { provider: string }',
248
270
  description:
249
- 'Choose one supported connection mechanism: token setup, a WebSocket relay, or a WebRTC SDP endpoint. Conversation semantics and connection capabilities are independent.',
271
+ 'Choose one supported connection mechanism: token setup, a WebSocket relay, a WebRTC SDP endpoint, or a provider-managed session endpoint. Conversation semantics and connection capabilities are independent.',
250
272
  properties: [
251
273
  {
252
274
  type: 'Object',
@@ -265,6 +287,13 @@ and completion of delegated work do not prove that the corresponding audio was h
265
287
  description:
266
288
  'Application WebRTC setup endpoint. Accepts JSON { sdp, sessionConfig } and returns JSON { sdp, sessionId }.',
267
289
  },
290
+ {
291
+ name: 'provider',
292
+ type: 'string',
293
+ isOptional: true,
294
+ description:
295
+ 'Application endpoint for a provider-owned transport. Mistral uses POST to create a session and DELETE with a sessionId query parameter to close it.',
296
+ },
268
297
  {
269
298
  name: 'websocket',
270
299
  type: 'string',
@@ -396,19 +425,19 @@ and completion of delegated work do not prove that the corresponding audio was h
396
425
  name: 'connect',
397
426
  type: '(options?: { stream?: MediaStream; capture?: boolean }) => Promise<void>',
398
427
  description:
399
- 'Starts the selected connection. Continuous browser sessions acquire microphone audio by default; capture: false opts out. A supplied Live stream remains caller-owned. The zero-argument overload remains usable as an event handler.',
428
+ 'Starts the selected connection. Continuous Live and Mistral sessions acquire microphone audio by default; capture: false opts out. Supplied tracks remain caller-owned on these paths. The zero-argument overload remains usable as an event handler.',
400
429
  },
401
430
  {
402
431
  name: 'close',
403
432
  type: '(options?: { eventId?: string }) => Promise<void>',
404
433
  description:
405
- 'Stops capture/submissions and waits for bounded graceful finalization where supported. Inspect session.finalization for confirmation; models without acknowledged close disconnect immediately.',
434
+ 'Stops capture/submissions and waits for bounded graceful finalization where supported. Provider-owned transports await session cleanup. For Live, inspect session.finalization for usage confirmation; models without acknowledged close disconnect immediately.',
406
435
  },
407
436
  {
408
437
  name: 'disconnect',
409
438
  type: '() => void',
410
439
  description:
411
- 'Immediately releases transports and owned media. Preserves latest usage as unconfirmed unless a terminal provider event was received.',
440
+ 'Immediately releases transports and owned media. Provider-owned transports initiate server cleanup without awaiting it. Preserves latest usage as unconfirmed unless a terminal provider event was received.',
412
441
  },
413
442
  {
414
443
  name: 'addToolOutput',
@@ -495,7 +495,7 @@ console.log(result.text);
495
495
  console.log(result.usage);
496
496
  ```
497
497
 
498
- Accepts `WorkflowAgentGenerateOptions`, sharing prompt/messages, instructions, generation settings, tools and runtime context, preparation, stopping, output, and lifecycle options with `stream()`. It does not accept `writable`, `includeRawChunks`, `experimental_transform`, `sendFinish`, or `preventClose`. Without a configured stop condition, generation stops after at most 20 steps. `stream()` retains its existing unbounded default.
498
+ Accepts `WorkflowAgentGenerateOptions`, sharing prompt/messages, instructions, generation settings, tools and runtime context, preparation, stopping, output, and lifecycle options with `stream()`. It does not accept `writable`, `includeRawChunks`, `sendFinish`, or `preventClose`. Without a configured stop condition, generation stops after at most 20 steps. `stream()` retains its existing unbounded default.
499
499
 
500
500
  The `include` option accepts `requestBody`, `responseBody`, and `requestMessages` booleans, all defaulting to `false`. Enable them only when the additional request/response data is needed. Response IDs, timestamps, model IDs, headers, and provider metadata are retained when supplied by the model.
501
501
 
@@ -626,12 +626,6 @@ const result = await agent.stream({
626
626
  isOptional: true,
627
627
  description: "Callback to attempt automatic recovery when a tool call cannot be parsed. Mark the function with 'use step' when used in a workflow.",
628
628
  },
629
- {
630
- name: 'experimental_transform',
631
- type: 'StreamTextTransform | Array<StreamTextTransform>',
632
- isOptional: true,
633
- description: "Factories for stream transformations, applied in order to raw provider events before model processing and writable output. Each factory must return a TransformStream synchronously and maintain the stream structure. Mark each factory with 'use step' when used in a workflow.",
634
- },
635
629
  {
636
630
  name: 'experimental_download',
637
631
  type: 'DownloadFunction',
@@ -817,48 +811,6 @@ The callback can be asynchronous. Return the repaired tool call, or `null`
817
811
  if the repair cannot be completed. See [tool call repair](/docs/ai-sdk-core/tools-and-tool-calling#tool-call-repair)
818
812
  for repair strategies.
819
813
 
820
- ### `experimental_transform` in a workflow
821
-
822
- Pass a transform factory, or an array of factories, to `stream()`.
823
- Mark each factory with `'use step'` so Workflow can load it inside the model
824
- step. The factory must return a `TransformStream` synchronously. Keep any
825
- captured values serializable.
826
-
827
- ```ts
828
- import type { LanguageModelV4StreamPart } from '@ai-sdk/provider';
829
- import { WorkflowAgent } from '@ai-sdk/workflow';
830
- import { getWritable } from 'workflow';
831
-
832
- function uppercaseText() {
833
- 'use step';
834
-
835
- return new TransformStream<
836
- LanguageModelV4StreamPart,
837
- LanguageModelV4StreamPart
838
- >({
839
- transform(part, controller) {
840
- controller.enqueue(
841
- part.type === 'text-delta'
842
- ? { ...part, delta: part.delta.toUpperCase() }
843
- : part,
844
- );
845
- },
846
- });
847
- }
848
-
849
- export async function uppercaseWorkflow() {
850
- 'use workflow';
851
-
852
- const agent = new WorkflowAgent({ model: 'openai/gpt-5' });
853
-
854
- return agent.stream({
855
- messages: [{ role: 'user', content: 'Write a short greeting.' }],
856
- writable: getWritable(),
857
- experimental_transform: uppercaseText,
858
- });
859
- }
860
- ```
861
-
862
814
  ## Utilities
863
815
 
864
816
  ### `createModelCallToUIChunkTransform(options?)`