ai 7.0.162 → 7.0.164
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +78 -0
- package/dist/index.d.ts +166 -41
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +539 -112
- package/dist/index.js.map +1 -1
- package/dist/internal/index.d.ts +43 -10
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +209 -24
- package/dist/internal/index.js.map +1 -1
- package/dist/test/index.d.ts +3 -1
- package/dist/test/index.d.ts.map +1 -1
- package/dist/test/index.js +2 -1
- package/dist/test/index.js.map +1 -1
- package/docs/02-foundations/02-providers-and-models.mdx +2 -0
- package/docs/03-agents/04-loop-control.mdx +34 -0
- package/docs/03-agents/06-memory.mdx +42 -0
- package/docs/03-ai-sdk-core/05-generating-text.mdx +1 -0
- package/docs/03-ai-sdk-core/16-mcp-tools.mdx +12 -0
- package/docs/03-ai-sdk-core/36-realtime.mdx +74 -1
- package/docs/03-ai-sdk-core/36-transcription.mdx +51 -4
- package/docs/03-ai-sdk-core/40-middleware.mdx +7 -2
- package/docs/03-ai-sdk-core/60-telemetry.mdx +37 -4
- package/docs/04-ai-sdk-ui/50-stream-protocol.mdx +13 -0
- package/docs/06-advanced/04-caching.mdx +13 -4
- package/docs/07-reference/01-ai-sdk-core/01-generate-text.mdx +33 -28
- package/docs/07-reference/01-ai-sdk-core/02-stream-text.mdx +58 -1
- package/docs/07-reference/01-ai-sdk-core/11-transcribe.mdx +17 -2
- package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx +8 -1
- package/docs/07-reference/01-ai-sdk-core/31-ui-message.mdx +23 -0
- package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +37 -8
- package/docs/07-reference/04-ai-sdk-workflow/01-workflow-agent.mdx +1 -49
- package/docs/08-migration-guides/23-migration-guide-7-0.mdx +36 -34
- package/package.json +12 -12
- package/src/generate-object/generate-object.ts +2 -1
- package/src/generate-object/stream-object.ts +7 -3
- package/src/generate-text/generate-text.ts +23 -3
- package/src/generate-text/is-output-chunk.ts +1 -0
- package/src/generate-text/language-model-events.ts +3 -0
- package/src/generate-text/output.ts +90 -37
- package/src/generate-text/smooth-stream.ts +5 -0
- package/src/generate-text/steering.ts +5 -4
- package/src/generate-text/stream-language-model-call.ts +16 -1
- package/src/generate-text/stream-text-result.ts +29 -1
- package/src/generate-text/stream-text.ts +25 -1
- package/src/generate-text/to-response-messages.ts +6 -2
- package/src/generate-video/generate-video-result.ts +11 -2
- package/src/generate-video/generate-video.ts +26 -2
- package/src/model/as-transcription-model-v4.ts +25 -0
- package/src/prompt/convert-to-language-model-prompt.ts +86 -14
- package/src/realtime/index.ts +3 -0
- package/src/realtime/realtime-attempt.ts +1 -0
- package/src/realtime/realtime-session.ts +142 -21
- package/src/telemetry/create-telemetry-dispatcher.ts +8 -1
- package/src/telemetry/filter-telemetry-event.ts +165 -0
- package/src/telemetry/get-request-origin.ts +25 -0
- package/src/telemetry/tracing-channel-publisher.ts +19 -4
- package/src/test/mock-transcription-model-v4.ts +4 -0
- package/src/transcribe/transcribe-result.ts +20 -0
- package/src/transcribe/transcribe.ts +107 -20
- package/src/transcribe/transcription-events.ts +28 -8
- package/src/types/language-model-request-metadata.ts +3 -0
- package/src/types/realtime-model.ts +9 -0
- package/src/ui/convert-to-model-messages.ts +36 -28
- package/src/ui/index.ts +1 -0
- package/src/ui/process-ui-message-stream.ts +10 -0
- package/src/ui/ui-messages.ts +20 -0
- package/src/ui/validate-ui-messages.ts +21 -0
- package/src/ui-message-stream/to-ui-message-chunk.ts +72 -1
- package/src/ui-message-stream/ui-message-chunks.ts +31 -0
|
@@ -74,6 +74,13 @@ By default, both inputs and outputs are recorded. You can disable them by settin
|
|
|
74
74
|
Disabling the recording of inputs and outputs can be useful for privacy, data transfer, and performance reasons.
|
|
75
75
|
You might for example want to disable recording inputs if they contain sensitive information.
|
|
76
76
|
|
|
77
|
+
These settings filter content before lifecycle events reach telemetry integrations
|
|
78
|
+
and Node.js `ai:telemetry` subscribers. Raw provider bodies and provider metadata
|
|
79
|
+
are omitted when either recording option is disabled. Raw tracing span results
|
|
80
|
+
are also omitted in that case because they can contain both inputs and outputs.
|
|
81
|
+
Tracing span errors use a generic failure message.
|
|
82
|
+
Application lifecycle callbacks and returned results retain their full content.
|
|
83
|
+
|
|
77
84
|
For `generateSpeech`, input telemetry contains the source text and output
|
|
78
85
|
telemetry contains the generated audio byte length, media type, and format.
|
|
79
86
|
For `transcribe` and `experimental_streamTranscribe`, telemetry contains the
|
|
@@ -102,6 +109,23 @@ such as `gen_ai.usage.characters`, `gen_ai.usage.seconds`, and
|
|
|
102
109
|
callbacks, and can also be included as the supplemental `ai.response.usage`
|
|
103
110
|
attribute by enabling `usage: true`.
|
|
104
111
|
|
|
112
|
+
### Provider endpoints
|
|
113
|
+
|
|
114
|
+
For completed language-model calls, providers can report the known HTTP request
|
|
115
|
+
endpoint. `OpenTelemetry` records its host as `server.address` and its port as
|
|
116
|
+
`server.port` on the `chat` span. This includes default ports (443 for HTTPS and
|
|
117
|
+
80 for HTTP) and works when input and output recording are disabled.
|
|
118
|
+
|
|
119
|
+
OpenAI Chat reports this information for both `generateText` and `streamText`,
|
|
120
|
+
including custom `baseURL` settings. Azure Chat and Bedrock Mantle Chat reuse
|
|
121
|
+
this adapter and also report their configured endpoint. Other providers leave these attributes
|
|
122
|
+
unset until they report their endpoint; no destination is inferred from the
|
|
123
|
+
provider or model name, including an unknown backend behind an intermediary.
|
|
124
|
+
|
|
125
|
+
The model-call end event's optional `request.url` contains only the HTTP origin.
|
|
126
|
+
Credentials, paths, queries, and fragments are removed before telemetry events
|
|
127
|
+
are emitted. The OpenTelemetry attributes contain only host and port.
|
|
128
|
+
|
|
105
129
|
### Opting out
|
|
106
130
|
|
|
107
131
|
Telemetry is opt-out. To disable telemetry for a specific call, set `isEnabled: false`:
|
|
@@ -594,7 +618,7 @@ metrics. The `LegacyOpenTelemetry` integration does not emit this histogram.
|
|
|
594
618
|
|
|
595
619
|
#### generateText / streamText
|
|
596
620
|
|
|
597
|
-
For `generateText` and `streamText`, the integration records
|
|
621
|
+
For `generateText` and `streamText`, the integration records 4 types of spans:
|
|
598
622
|
|
|
599
623
|
- **`invoke_agent {modelId}`** (default root span name, `INTERNAL`): covers the full operation including all steps and tool calls. Use [`agentSpanName`](#agent-span-names) to customize the name.
|
|
600
624
|
|
|
@@ -622,7 +646,16 @@ For `generateText` and `streamText`, the integration records 3 types of spans:
|
|
|
622
646
|
- `gen_ai.usage.cache_creation.input_tokens`: cached input tokens created
|
|
623
647
|
- `gen_ai.output.messages`: the output in [GenAI SemConv message format](#genai-message-format) (when `recordOutputs` is enabled)
|
|
624
648
|
|
|
625
|
-
- **`
|
|
649
|
+
- **`step {n}`** (step span, `INTERNAL`): one span per step, nested under the root span. `n` is the step number, starting at 1. It contains the model call and tool spans of that step.
|
|
650
|
+
|
|
651
|
+
Initial attributes:
|
|
652
|
+
- `gen_ai.operation.name`: `"agent_step"`
|
|
653
|
+
|
|
654
|
+
`agent_step` is an AI SDK-defined custom operation name, not a well-known
|
|
655
|
+
OpenTelemetry GenAI operation. It marks an orchestration step, while the
|
|
656
|
+
nested `chat` span marks the model call.
|
|
657
|
+
|
|
658
|
+
- **`chat {modelId}`** (model call span, `CLIENT`): one span per LLM provider call, nested under the step span.
|
|
626
659
|
|
|
627
660
|
Initial attributes:
|
|
628
661
|
- `gen_ai.operation.name`: `"chat"`
|
|
@@ -655,11 +688,11 @@ For `generateText` and `streamText`, the integration records 3 types of spans:
|
|
|
655
688
|
`streamText` with the `output` property instead.
|
|
656
689
|
</Note>
|
|
657
690
|
|
|
658
|
-
The deprecated object APIs emit the same
|
|
691
|
+
The deprecated object APIs emit the same spans as `generateText`/`streamText`, except that there is no `step {n}` span: the `chat {modelId}` span is nested directly under the root span. The root span has these additional attributes:
|
|
659
692
|
|
|
660
693
|
- `gen_ai.output.type`: `"json"`
|
|
661
694
|
|
|
662
|
-
The
|
|
695
|
+
The `chat {modelId}` spans also include `gen_ai.output.type: "json"`, and `gen_ai.output.messages` contains the generated object as a text part.
|
|
663
696
|
|
|
664
697
|
#### embed / embedMany
|
|
665
698
|
|
|
@@ -425,6 +425,19 @@ data: {"type":"tool-output-denied","toolCallId":"call_fJdQDqnXeGxTmr4E3YPSR7Ar"}
|
|
|
425
425
|
|
|
426
426
|
```
|
|
427
427
|
|
|
428
|
+
### Steering Message Part
|
|
429
|
+
|
|
430
|
+
Experimental. A user message that was added to the run with [steering](/docs/agents/loop-control#steering). It is sent before the `start-step` part of the step that received it, and `useChat` adds it to the assistant message as a `steering-message` part.
|
|
431
|
+
|
|
432
|
+
Format: Server-Sent Event with JSON object
|
|
433
|
+
|
|
434
|
+
Example:
|
|
435
|
+
|
|
436
|
+
```
|
|
437
|
+
data: {"type":"steering-message","id":"steer-1","parts":[{"type":"text","text":"I meant Paris, Texas."}]}
|
|
438
|
+
|
|
439
|
+
```
|
|
440
|
+
|
|
428
441
|
### Start Step Part
|
|
429
442
|
|
|
430
443
|
A part indicating the start of a step.
|
|
@@ -14,6 +14,7 @@ and the [`simulateReadableStream`](/docs/reference/ai-sdk-core/simulate-readable
|
|
|
14
14
|
|
|
15
15
|
Language model middleware is a way to enhance the behavior of language models by intercepting and modifying the calls to the language model.
|
|
16
16
|
Let's see how you can use language model middleware to cache responses.
|
|
17
|
+
Include the provider and model ID in the cache key alongside the call parameters to keep cached responses separate across models and providers.
|
|
17
18
|
|
|
18
19
|
```ts filename="ai/middleware.ts"
|
|
19
20
|
import { Redis } from '@upstash/redis';
|
|
@@ -31,8 +32,12 @@ const redis = new Redis({
|
|
|
31
32
|
|
|
32
33
|
export const cacheMiddleware: LanguageModelV4Middleware = {
|
|
33
34
|
specificationVersion: 'v4',
|
|
34
|
-
wrapGenerate: async ({ doGenerate, params }) => {
|
|
35
|
-
const cacheKey = JSON.stringify(
|
|
35
|
+
wrapGenerate: async ({ doGenerate, params, model }) => {
|
|
36
|
+
const cacheKey = JSON.stringify({
|
|
37
|
+
provider: model.provider,
|
|
38
|
+
modelId: model.modelId,
|
|
39
|
+
params,
|
|
40
|
+
});
|
|
36
41
|
|
|
37
42
|
const cached = (await redis.get(cacheKey)) as Awaited<
|
|
38
43
|
ReturnType<LanguageModelV4['doGenerate']>
|
|
@@ -56,8 +61,12 @@ export const cacheMiddleware: LanguageModelV4Middleware = {
|
|
|
56
61
|
|
|
57
62
|
return result;
|
|
58
63
|
},
|
|
59
|
-
wrapStream: async ({ doStream, params }) => {
|
|
60
|
-
const cacheKey = JSON.stringify(
|
|
64
|
+
wrapStream: async ({ doStream, params, model }) => {
|
|
65
|
+
const cacheKey = JSON.stringify({
|
|
66
|
+
provider: model.provider,
|
|
67
|
+
modelId: model.modelId,
|
|
68
|
+
params,
|
|
69
|
+
});
|
|
61
70
|
|
|
62
71
|
// Check if the result is in the cache
|
|
63
72
|
const cached = await redis.get(cacheKey);
|
|
@@ -1277,6 +1277,11 @@ To see `generateText` in action, check out [these examples](#examples).
|
|
|
1277
1277
|
{
|
|
1278
1278
|
type: 'GenerateTextStepStartEvent',
|
|
1279
1279
|
parameters: [
|
|
1280
|
+
{
|
|
1281
|
+
name: 'callId',
|
|
1282
|
+
type: 'string',
|
|
1283
|
+
description: 'Unique identifier for the generation call.',
|
|
1284
|
+
},
|
|
1280
1285
|
{
|
|
1281
1286
|
name: 'stepNumber',
|
|
1282
1287
|
type: 'number',
|
|
@@ -1339,40 +1344,12 @@ To see `generateText` in action, check out [these examples](#examples).
|
|
|
1339
1344
|
description:
|
|
1340
1345
|
'Additional provider-specific options for this step.',
|
|
1341
1346
|
},
|
|
1342
|
-
{
|
|
1343
|
-
name: 'timeout',
|
|
1344
|
-
type: 'number | { totalMs?: number; stepMs?: number; chunkMs?: number } | undefined',
|
|
1345
|
-
description:
|
|
1346
|
-
'Timeout configuration for the generation. Can be a number (milliseconds) or an object with totalMs, stepMs, chunkMs.',
|
|
1347
|
-
},
|
|
1348
|
-
{
|
|
1349
|
-
name: 'headers',
|
|
1350
|
-
type: 'Record<string, string | undefined> | undefined',
|
|
1351
|
-
description: 'Additional HTTP headers sent with the request.',
|
|
1352
|
-
},
|
|
1353
|
-
{
|
|
1354
|
-
name: 'stopWhen',
|
|
1355
|
-
type: 'StopCondition<TOOLS> | Array<StopCondition<TOOLS>> | undefined',
|
|
1356
|
-
description:
|
|
1357
|
-
'Condition(s) for stopping the generation. When the condition is an array, any of the conditions can be met to stop.',
|
|
1358
|
-
},
|
|
1359
1347
|
{
|
|
1360
1348
|
name: 'output',
|
|
1361
1349
|
type: 'OUTPUT | undefined',
|
|
1362
1350
|
description:
|
|
1363
1351
|
'The output specification for structured outputs, if configured.',
|
|
1364
1352
|
},
|
|
1365
|
-
{
|
|
1366
|
-
name: 'abortSignal',
|
|
1367
|
-
type: 'AbortSignal | undefined',
|
|
1368
|
-
description: 'Abort signal for cancelling the operation.',
|
|
1369
|
-
},
|
|
1370
|
-
{
|
|
1371
|
-
name: 'include',
|
|
1372
|
-
type: '{ requestBody?: boolean; requestMessages?: boolean; responseBody?: boolean } | undefined',
|
|
1373
|
-
description:
|
|
1374
|
-
'Settings for controlling what data is included in step results.',
|
|
1375
|
-
},
|
|
1376
1353
|
{
|
|
1377
1354
|
name: 'runtimeContext',
|
|
1378
1355
|
type: 'CONTEXT',
|
|
@@ -1930,6 +1907,13 @@ To see `generateText` in action, check out [these examples](#examples).
|
|
|
1930
1907
|
{
|
|
1931
1908
|
type: 'LanguageModelRequestMetadata',
|
|
1932
1909
|
parameters: [
|
|
1910
|
+
{
|
|
1911
|
+
name: 'url',
|
|
1912
|
+
type: 'string',
|
|
1913
|
+
isOptional: true,
|
|
1914
|
+
description:
|
|
1915
|
+
'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
|
|
1916
|
+
},
|
|
1933
1917
|
{
|
|
1934
1918
|
name: 'messages',
|
|
1935
1919
|
type: 'Array<ModelMessage>',
|
|
@@ -2386,6 +2370,13 @@ To see `generateText` in action, check out [these examples](#examples).
|
|
|
2386
2370
|
{
|
|
2387
2371
|
type: 'LanguageModelRequestMetadata',
|
|
2388
2372
|
parameters: [
|
|
2373
|
+
{
|
|
2374
|
+
name: 'url',
|
|
2375
|
+
type: 'string',
|
|
2376
|
+
isOptional: true,
|
|
2377
|
+
description:
|
|
2378
|
+
'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
|
|
2379
|
+
},
|
|
2389
2380
|
{
|
|
2390
2381
|
name: 'messages',
|
|
2391
2382
|
type: 'Array<ModelMessage>',
|
|
@@ -2811,6 +2802,13 @@ To see `generateText` in action, check out [these examples](#examples).
|
|
|
2811
2802
|
{
|
|
2812
2803
|
type: 'LanguageModelRequestMetadata',
|
|
2813
2804
|
parameters: [
|
|
2805
|
+
{
|
|
2806
|
+
name: 'url',
|
|
2807
|
+
type: 'string',
|
|
2808
|
+
isOptional: true,
|
|
2809
|
+
description:
|
|
2810
|
+
'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
|
|
2811
|
+
},
|
|
2814
2812
|
{
|
|
2815
2813
|
name: 'messages',
|
|
2816
2814
|
type: 'Array<ModelMessage>',
|
|
@@ -3192,6 +3190,13 @@ To see `generateText` in action, check out [these examples](#examples).
|
|
|
3192
3190
|
{
|
|
3193
3191
|
type: 'LanguageModelRequestMetadata',
|
|
3194
3192
|
parameters: [
|
|
3193
|
+
{
|
|
3194
|
+
name: 'url',
|
|
3195
|
+
type: 'string',
|
|
3196
|
+
isOptional: true,
|
|
3197
|
+
description:
|
|
3198
|
+
'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
|
|
3199
|
+
},
|
|
3195
3200
|
{
|
|
3196
3201
|
name: 'messages',
|
|
3197
3202
|
type: 'Array<ModelMessage>',
|
|
@@ -1032,7 +1032,7 @@ retain the content their contracts require.
|
|
|
1032
1032
|
name: 'chunk',
|
|
1033
1033
|
type: 'TextStreamPart<TOOLS>',
|
|
1034
1034
|
description:
|
|
1035
|
-
'The stream part. This is the same union as `stream` and includes `start`, `start-step`, `text-start`, `text-delta`, `text-end`, `reasoning-start`, `reasoning-delta`, `reasoning-end`, `custom`, `source`, `file`, `reasoning-file`, `tool-call`, `tool-input-start`, `tool-input-delta`, `tool-input-end`, `tool-result`, `tool-error`, `tool-output-denied`, `tool-approval-request`, `tool-approval-response`, `finish-step`, `finish`, `abort`, `error`, and `
|
|
1035
|
+
'The stream part. This is the same union as `stream` and includes `start`, `steering-message`, `start-step`, `text-start`, `text-delta`, `text-end`, `reasoning-start`, `reasoning-delta`, `reasoning-end`, `custom`, `source`, `file`, `reasoning-file`, `tool-call`, `tool-input-start`, `tool-input-delta`, `tool-input-end`, `tool-result`, `tool-error`, `tool-output-denied`, `tool-approval-request`, `tool-approval-response`, `finish-step`, `finish`, `abort`, `error`, `raw`, and `ping`.',
|
|
1036
1036
|
properties: [
|
|
1037
1037
|
{
|
|
1038
1038
|
type: 'TextStreamPart',
|
|
@@ -3017,6 +3017,12 @@ retain the content their contracts require.
|
|
|
3017
3017
|
{
|
|
3018
3018
|
type: 'LanguageModelRequestMetadata',
|
|
3019
3019
|
parameters: [
|
|
3020
|
+
{
|
|
3021
|
+
name: 'url',
|
|
3022
|
+
type: 'string',
|
|
3023
|
+
isOptional: true,
|
|
3024
|
+
description: 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
|
|
3025
|
+
},
|
|
3020
3026
|
{
|
|
3021
3027
|
name: 'messages',
|
|
3022
3028
|
type: 'Array<ModelMessage>',
|
|
@@ -3309,6 +3315,13 @@ retain the content their contracts require.
|
|
|
3309
3315
|
{
|
|
3310
3316
|
type: 'RequestMetadata',
|
|
3311
3317
|
parameters: [
|
|
3318
|
+
{
|
|
3319
|
+
name: 'url',
|
|
3320
|
+
type: 'string',
|
|
3321
|
+
isOptional: true,
|
|
3322
|
+
description:
|
|
3323
|
+
'The HTTP origin reported by the provider, without URL credentials, paths, queries or fragments.',
|
|
3324
|
+
},
|
|
3312
3325
|
{
|
|
3313
3326
|
name: 'messages',
|
|
3314
3327
|
type: 'Array<ModelMessage>',
|
|
@@ -3418,6 +3431,17 @@ retain the content their contracts require.
|
|
|
3418
3431
|
description:
|
|
3419
3432
|
'A stream with all events, including text deltas, tool calls, tool results, and errors. Well-formed mid-stream provider failures are emitted in error parts as StreamProviderError instances. You can use it as either an AsyncIterable or a ReadableStream. Only errors that stop the stream, such as network errors, are thrown.',
|
|
3420
3433
|
properties: [
|
|
3434
|
+
{
|
|
3435
|
+
type: 'TextStreamPingPart',
|
|
3436
|
+
parameters: [
|
|
3437
|
+
{
|
|
3438
|
+
name: 'type',
|
|
3439
|
+
type: "'ping'",
|
|
3440
|
+
description:
|
|
3441
|
+
'A transient provider heartbeat. It contains no generated content, does not reset model-output timeouts, and is omitted from text and UI message streams. Providers may require an explicit opt-in.',
|
|
3442
|
+
},
|
|
3443
|
+
],
|
|
3444
|
+
},
|
|
3421
3445
|
{
|
|
3422
3446
|
type: 'TextStreamTextEndPart',
|
|
3423
3447
|
description: 'End of a text block, including optional citations supporting that block.',
|
|
@@ -3677,6 +3701,33 @@ retain the content their contracts require.
|
|
|
3677
3701
|
},
|
|
3678
3702
|
],
|
|
3679
3703
|
},
|
|
3704
|
+
{
|
|
3705
|
+
type: 'TextStreamPart',
|
|
3706
|
+
parameters: [
|
|
3707
|
+
{
|
|
3708
|
+
name: 'type',
|
|
3709
|
+
type: "'steering-message'",
|
|
3710
|
+
description:
|
|
3711
|
+
'Experimental. A user message that was added to the run with `experimental_steering`. It is streamed before the `start-step` part of the step that received it.',
|
|
3712
|
+
},
|
|
3713
|
+
{
|
|
3714
|
+
name: 'id',
|
|
3715
|
+
type: 'string',
|
|
3716
|
+
description: 'The id of the steering message.',
|
|
3717
|
+
},
|
|
3718
|
+
{
|
|
3719
|
+
name: 'message',
|
|
3720
|
+
type: 'UserModelMessage',
|
|
3721
|
+
description: 'The user message that was added to the run.',
|
|
3722
|
+
},
|
|
3723
|
+
{
|
|
3724
|
+
name: 'stepNumber',
|
|
3725
|
+
type: 'number',
|
|
3726
|
+
description:
|
|
3727
|
+
'The number of the step whose model call received the message.',
|
|
3728
|
+
},
|
|
3729
|
+
],
|
|
3730
|
+
},
|
|
3680
3731
|
{
|
|
3681
3732
|
type: 'TextStreamPart',
|
|
3682
3733
|
parameters: [
|
|
@@ -3694,6 +3745,12 @@ retain the content their contracts require.
|
|
|
3694
3745
|
{
|
|
3695
3746
|
type: 'LanguageModelRequestMetadata',
|
|
3696
3747
|
parameters: [
|
|
3748
|
+
{
|
|
3749
|
+
name: 'url',
|
|
3750
|
+
type: 'string',
|
|
3751
|
+
isOptional: true,
|
|
3752
|
+
description: 'HTTP origin reported by the provider, without URL credentials, paths, queries or fragments. Unset when unavailable.',
|
|
3753
|
+
},
|
|
3697
3754
|
{
|
|
3698
3755
|
name: 'messages',
|
|
3699
3756
|
type: 'Array<ModelMessage>',
|
|
@@ -38,7 +38,15 @@ console.log(transcript);
|
|
|
38
38
|
{
|
|
39
39
|
name: 'audio',
|
|
40
40
|
type: 'DataContent (string | Uint8Array | ArrayBuffer | Buffer) | URL',
|
|
41
|
-
description:
|
|
41
|
+
description:
|
|
42
|
+
'The audio file to generate the transcript from. HTTP(S) URLs are sent directly to models with native URL support unless a custom download function is supplied; other models download the audio locally.',
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
name: 'timestampGranularities',
|
|
46
|
+
type: "Array<'word' | 'segment'>",
|
|
47
|
+
isOptional: true,
|
|
48
|
+
description:
|
|
49
|
+
'Word and segment timings to request. Supported by the OpenAI, Azure Speech, Vertex Gemini, and xAI adapters. Explicit provider options take precedence; these adapters warn about unavailable requested timings. AI Gateway warns that it does not forward this option; use provider options or a direct provider model. Omit this option to keep provider defaults. An empty array requests no additional timing detail; it does not suppress timings returned by default.',
|
|
42
50
|
},
|
|
43
51
|
{
|
|
44
52
|
name: 'providerOptions',
|
|
@@ -76,7 +84,7 @@ console.log(transcript);
|
|
|
76
84
|
type: '(options: { url: URL; abortSignal?: AbortSignal }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>',
|
|
77
85
|
isOptional: true,
|
|
78
86
|
description:
|
|
79
|
-
'Custom download function for fetching audio from URLs. Use `createDownload()` from `ai` to create a download function with custom size limits, e.g. `createDownload({ maxBytes: 50 * 1024 * 1024 })`. Default: built-in download with 2 GiB limit.',
|
|
87
|
+
'Custom download function for fetching audio from URLs. Supplying this option forces local downloading even for models with native URL support. Use `createDownload()` from `ai` to create a download function with custom size limits, e.g. `createDownload({ maxBytes: 50 * 1024 * 1024 })`. Default: native URL transcription when supported, otherwise built-in download with 2 GiB limit.',
|
|
80
88
|
},
|
|
81
89
|
]}
|
|
82
90
|
/>
|
|
@@ -96,6 +104,13 @@ console.log(transcript);
|
|
|
96
104
|
description:
|
|
97
105
|
'An array of transcript segments, each containing a portion of the transcribed text along with its start and end times in seconds.',
|
|
98
106
|
},
|
|
107
|
+
{
|
|
108
|
+
name: 'words',
|
|
109
|
+
type: 'Array<{ text: string; startSecond: number; endSecond: number }>',
|
|
110
|
+
isOptional: true,
|
|
111
|
+
description:
|
|
112
|
+
'Word-level timings when the adapter exposes them, including timings returned by default. Undefined when unavailable. Existing segments behavior is preserved: some providers also put word timings in segments.',
|
|
113
|
+
},
|
|
99
114
|
{
|
|
100
115
|
name: 'language',
|
|
101
116
|
type: 'string | undefined',
|
|
@@ -220,7 +220,8 @@ console.log(videos);
|
|
|
220
220
|
{
|
|
221
221
|
name: 'video',
|
|
222
222
|
type: 'GeneratedFile',
|
|
223
|
-
description:
|
|
223
|
+
description:
|
|
224
|
+
'The first video available to the SDK. For upload-only results, accessing this property throws NoVideoGeneratedError; use uploadedVideos instead.',
|
|
224
225
|
properties: [
|
|
225
226
|
{
|
|
226
227
|
type: 'GeneratedFile',
|
|
@@ -273,6 +274,12 @@ console.log(videos);
|
|
|
273
274
|
},
|
|
274
275
|
],
|
|
275
276
|
},
|
|
277
|
+
{
|
|
278
|
+
name: 'uploadedVideos',
|
|
279
|
+
type: 'number',
|
|
280
|
+
description:
|
|
281
|
+
'Number of videos delivered directly to caller-managed storage by the provider. These are not downloaded and are not included in videos.',
|
|
282
|
+
},
|
|
276
283
|
{
|
|
277
284
|
name: 'warnings',
|
|
278
285
|
type: 'Warning[]',
|
|
@@ -320,6 +320,29 @@ type DataUIPart<DATA_TYPES extends UIDataTypes> = ValueOf<{
|
|
|
320
320
|
}>;
|
|
321
321
|
```
|
|
322
322
|
|
|
323
|
+
### `SteeringMessageUIPart`
|
|
324
|
+
|
|
325
|
+
<Note>
|
|
326
|
+
This part is experimental and exported as
|
|
327
|
+
`Experimental_SteeringMessageUIPart`.
|
|
328
|
+
</Note>
|
|
329
|
+
|
|
330
|
+
A user message that was added to the run with [steering](/docs/agents/loop-control#steering) while the assistant message was generated. It is placed before the `step-start` part of the step that received it.
|
|
331
|
+
|
|
332
|
+
```typescript
|
|
333
|
+
type SteeringMessageUIPart = {
|
|
334
|
+
type: 'steering-message';
|
|
335
|
+
/**
|
|
336
|
+
* The id of the steering message.
|
|
337
|
+
*/
|
|
338
|
+
id: string;
|
|
339
|
+
/**
|
|
340
|
+
* The content of the user message.
|
|
341
|
+
*/
|
|
342
|
+
parts: Array<TextUIPart | FileUIPart>;
|
|
343
|
+
};
|
|
344
|
+
```
|
|
345
|
+
|
|
323
346
|
### `StepStartUIPart`
|
|
324
347
|
|
|
325
348
|
A step boundary part of a message.
|
|
@@ -13,7 +13,8 @@ Creates a browser-side realtime session for bidirectional audio and text
|
|
|
13
13
|
conversations with a realtime provider model.
|
|
14
14
|
|
|
15
15
|
The hook supports token-based provider WebSockets, application-owned WebSocket
|
|
16
|
-
relays,
|
|
16
|
+
relays, optional WebRTC, and provider-owned transports where the model declares
|
|
17
|
+
support. It provides controls for capture and playback, plus supported turn-based text input
|
|
17
18
|
and tool output. Turn-based conversation messages use `UIMessage[]`; continuous
|
|
18
19
|
transcript fragments remain in `session`.
|
|
19
20
|
|
|
@@ -184,12 +185,33 @@ updates remain unsupported.
|
|
|
184
185
|
|
|
185
186
|
## Lifecycle and failure handling
|
|
186
187
|
|
|
188
|
+
### Provider-owned sessions
|
|
189
|
+
|
|
190
|
+
For [Mistral voice agents](/providers/ai-sdk-providers/mistral#realtime-voice-agents),
|
|
191
|
+
use `api: { provider: '/api/mistral-realtime' }`. The Mistral provider handles
|
|
192
|
+
LiveKit internally.
|
|
193
|
+
The hook calls `POST` on that endpoint to create a session and
|
|
194
|
+
`DELETE ?sessionId=...` to close it. The server uses
|
|
195
|
+
`mistral.experimental_realtime.createSession()` and `closeSession()` with its API
|
|
196
|
+
key. Keep the model, voice, instructions, and allowed settings on the server.
|
|
197
|
+
|
|
198
|
+
`connect()` starts microphone capture and remote audio playback. Mistral
|
|
199
|
+
transcripts populate `messages`; audio travels through the provider's LiveKit
|
|
200
|
+
transport. Await `close()` to finish server-side cleanup. `disconnect()` and
|
|
201
|
+
unmount release local media and initiate cleanup without waiting. Client tools
|
|
202
|
+
use `onToolCall` and `addToolOutput`. Text, manual audio-buffer, and response
|
|
203
|
+
commands are unsupported. See the provider guide for the complete route and
|
|
204
|
+
client example.
|
|
205
|
+
|
|
206
|
+
### Shared lifecycle
|
|
207
|
+
|
|
187
208
|
Use `status === 'connected'` as the readiness signal; `connect()` does not promise
|
|
188
209
|
to wait for provider readiness. Operational startup errors are reported through
|
|
189
210
|
`onError` and `status`; the legacy resolve-and-report behavior is preserved.
|
|
190
211
|
|
|
191
|
-
`close()` stops local capture and submissions, then
|
|
192
|
-
confirms final usage or the close deadline
|
|
212
|
+
For continuous Live sessions, `close()` stops local capture and submissions, then
|
|
213
|
+
drains events until the provider confirms final usage or the close deadline
|
|
214
|
+
expires. A failed close-command send
|
|
193
215
|
uses the shorter accepted-event drain rather than waiting for an acknowledgment
|
|
194
216
|
to an unsent command. Read `session.finalization`: a fulfilled close promise does
|
|
195
217
|
not by itself confirm usage. `disconnect()` and unmount release resources immediately.
|
|
@@ -244,9 +266,9 @@ and completion of delegated work do not prove that the corresponding audio was h
|
|
|
244
266
|
},
|
|
245
267
|
{
|
|
246
268
|
name: 'api',
|
|
247
|
-
type: '{ token: string } | { websocket: string; protocols?: string[] } | { session: string }',
|
|
269
|
+
type: '{ token: string } | { websocket: string; protocols?: string[] } | { session: string } | { provider: string }',
|
|
248
270
|
description:
|
|
249
|
-
'Choose one supported connection mechanism: token setup, a WebSocket relay,
|
|
271
|
+
'Choose one supported connection mechanism: token setup, a WebSocket relay, a WebRTC SDP endpoint, or a provider-managed session endpoint. Conversation semantics and connection capabilities are independent.',
|
|
250
272
|
properties: [
|
|
251
273
|
{
|
|
252
274
|
type: 'Object',
|
|
@@ -265,6 +287,13 @@ and completion of delegated work do not prove that the corresponding audio was h
|
|
|
265
287
|
description:
|
|
266
288
|
'Application WebRTC setup endpoint. Accepts JSON { sdp, sessionConfig } and returns JSON { sdp, sessionId }.',
|
|
267
289
|
},
|
|
290
|
+
{
|
|
291
|
+
name: 'provider',
|
|
292
|
+
type: 'string',
|
|
293
|
+
isOptional: true,
|
|
294
|
+
description:
|
|
295
|
+
'Application endpoint for a provider-owned transport. Mistral uses POST to create a session and DELETE with a sessionId query parameter to close it.',
|
|
296
|
+
},
|
|
268
297
|
{
|
|
269
298
|
name: 'websocket',
|
|
270
299
|
type: 'string',
|
|
@@ -396,19 +425,19 @@ and completion of delegated work do not prove that the corresponding audio was h
|
|
|
396
425
|
name: 'connect',
|
|
397
426
|
type: '(options?: { stream?: MediaStream; capture?: boolean }) => Promise<void>',
|
|
398
427
|
description:
|
|
399
|
-
'Starts the selected connection. Continuous
|
|
428
|
+
'Starts the selected connection. Continuous Live and Mistral sessions acquire microphone audio by default; capture: false opts out. Supplied tracks remain caller-owned on these paths. The zero-argument overload remains usable as an event handler.',
|
|
400
429
|
},
|
|
401
430
|
{
|
|
402
431
|
name: 'close',
|
|
403
432
|
type: '(options?: { eventId?: string }) => Promise<void>',
|
|
404
433
|
description:
|
|
405
|
-
'Stops capture/submissions and waits for bounded graceful finalization where supported.
|
|
434
|
+
'Stops capture/submissions and waits for bounded graceful finalization where supported. Provider-owned transports await session cleanup. For Live, inspect session.finalization for usage confirmation; models without acknowledged close disconnect immediately.',
|
|
406
435
|
},
|
|
407
436
|
{
|
|
408
437
|
name: 'disconnect',
|
|
409
438
|
type: '() => void',
|
|
410
439
|
description:
|
|
411
|
-
'Immediately releases transports and owned media. Preserves latest usage as unconfirmed unless a terminal provider event was received.',
|
|
440
|
+
'Immediately releases transports and owned media. Provider-owned transports initiate server cleanup without awaiting it. Preserves latest usage as unconfirmed unless a terminal provider event was received.',
|
|
412
441
|
},
|
|
413
442
|
{
|
|
414
443
|
name: 'addToolOutput',
|
|
@@ -495,7 +495,7 @@ console.log(result.text);
|
|
|
495
495
|
console.log(result.usage);
|
|
496
496
|
```
|
|
497
497
|
|
|
498
|
-
Accepts `WorkflowAgentGenerateOptions`, sharing prompt/messages, instructions, generation settings, tools and runtime context, preparation, stopping, output, and lifecycle options with `stream()`. It does not accept `writable`, `includeRawChunks`, `
|
|
498
|
+
Accepts `WorkflowAgentGenerateOptions`, sharing prompt/messages, instructions, generation settings, tools and runtime context, preparation, stopping, output, and lifecycle options with `stream()`. It does not accept `writable`, `includeRawChunks`, `sendFinish`, or `preventClose`. Without a configured stop condition, generation stops after at most 20 steps. `stream()` retains its existing unbounded default.
|
|
499
499
|
|
|
500
500
|
The `include` option accepts `requestBody`, `responseBody`, and `requestMessages` booleans, all defaulting to `false`. Enable them only when the additional request/response data is needed. Response IDs, timestamps, model IDs, headers, and provider metadata are retained when supplied by the model.
|
|
501
501
|
|
|
@@ -626,12 +626,6 @@ const result = await agent.stream({
|
|
|
626
626
|
isOptional: true,
|
|
627
627
|
description: "Callback to attempt automatic recovery when a tool call cannot be parsed. Mark the function with 'use step' when used in a workflow.",
|
|
628
628
|
},
|
|
629
|
-
{
|
|
630
|
-
name: 'experimental_transform',
|
|
631
|
-
type: 'StreamTextTransform | Array<StreamTextTransform>',
|
|
632
|
-
isOptional: true,
|
|
633
|
-
description: "Factories for stream transformations, applied in order to raw provider events before model processing and writable output. Each factory must return a TransformStream synchronously and maintain the stream structure. Mark each factory with 'use step' when used in a workflow.",
|
|
634
|
-
},
|
|
635
629
|
{
|
|
636
630
|
name: 'experimental_download',
|
|
637
631
|
type: 'DownloadFunction',
|
|
@@ -817,48 +811,6 @@ The callback can be asynchronous. Return the repaired tool call, or `null`
|
|
|
817
811
|
if the repair cannot be completed. See [tool call repair](/docs/ai-sdk-core/tools-and-tool-calling#tool-call-repair)
|
|
818
812
|
for repair strategies.
|
|
819
813
|
|
|
820
|
-
### `experimental_transform` in a workflow
|
|
821
|
-
|
|
822
|
-
Pass a transform factory, or an array of factories, to `stream()`.
|
|
823
|
-
Mark each factory with `'use step'` so Workflow can load it inside the model
|
|
824
|
-
step. The factory must return a `TransformStream` synchronously. Keep any
|
|
825
|
-
captured values serializable.
|
|
826
|
-
|
|
827
|
-
```ts
|
|
828
|
-
import type { LanguageModelV4StreamPart } from '@ai-sdk/provider';
|
|
829
|
-
import { WorkflowAgent } from '@ai-sdk/workflow';
|
|
830
|
-
import { getWritable } from 'workflow';
|
|
831
|
-
|
|
832
|
-
function uppercaseText() {
|
|
833
|
-
'use step';
|
|
834
|
-
|
|
835
|
-
return new TransformStream<
|
|
836
|
-
LanguageModelV4StreamPart,
|
|
837
|
-
LanguageModelV4StreamPart
|
|
838
|
-
>({
|
|
839
|
-
transform(part, controller) {
|
|
840
|
-
controller.enqueue(
|
|
841
|
-
part.type === 'text-delta'
|
|
842
|
-
? { ...part, delta: part.delta.toUpperCase() }
|
|
843
|
-
: part,
|
|
844
|
-
);
|
|
845
|
-
},
|
|
846
|
-
});
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
export async function uppercaseWorkflow() {
|
|
850
|
-
'use workflow';
|
|
851
|
-
|
|
852
|
-
const agent = new WorkflowAgent({ model: 'openai/gpt-5' });
|
|
853
|
-
|
|
854
|
-
return agent.stream({
|
|
855
|
-
messages: [{ role: 'user', content: 'Write a short greeting.' }],
|
|
856
|
-
writable: getWritable(),
|
|
857
|
-
experimental_transform: uppercaseText,
|
|
858
|
-
});
|
|
859
|
-
}
|
|
860
|
-
```
|
|
861
|
-
|
|
862
814
|
## Utilities
|
|
863
815
|
|
|
864
816
|
### `createModelCallToUIChunkTransform(options?)`
|