ai 6.0.220 → 6.0.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/index.d.mts +22 -3
- package/dist/index.d.ts +22 -3
- package/dist/index.js +102 -44
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +101 -44
- package/dist/index.mjs.map +1 -1
- package/dist/internal/index.js +1 -1
- package/dist/internal/index.mjs +1 -1
- package/docs/02-foundations/02-providers-and-models.mdx +5 -0
- package/docs/03-ai-sdk-core/38-video-generation.mdx +35 -16
- package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx +3 -2
- package/package.json +4 -4
- package/src/generate-text/generate-text.ts +6 -4
- package/src/generate-text/output.ts +4 -1
- package/src/generate-text/run-tools-transformation.ts +3 -7
- package/src/generate-text/stream-text.ts +24 -13
- package/src/generate-video/generate-video.ts +97 -25
- package/src/ui/chat.ts +1 -1
- package/src/ui/convert-to-model-messages.ts +7 -5
- package/src/ui/index.ts +1 -0
- package/src/util/set-abort-timeout.ts +37 -0
package/dist/internal/index.js
CHANGED
|
@@ -152,7 +152,7 @@ function detectMediaType({
|
|
|
152
152
|
var import_provider_utils2 = require("@ai-sdk/provider-utils");
|
|
153
153
|
|
|
154
154
|
// src/version.ts
|
|
155
|
-
var VERSION = true ? "6.0.
|
|
155
|
+
var VERSION = true ? "6.0.222" : "0.0.0-test";
|
|
156
156
|
|
|
157
157
|
// src/util/download/download.ts
|
|
158
158
|
var download = async ({
|
package/dist/internal/index.mjs
CHANGED
|
@@ -111,10 +111,15 @@ Here are the capabilities of popular models:
|
|
|
111
111
|
|
|
112
112
|
| Provider | Model | Image Input | Object Generation | Tool Usage | Tool Streaming |
|
|
113
113
|
| -------------------------------------------------- | ------------------------------------------- | ------------------- | ------------------- | ------------------- | ------------------- |
|
|
114
|
+
| [xAI Grok](/providers/ai-sdk-providers/xai) | `grok-4.5` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
114
115
|
| [xAI Grok](/providers/ai-sdk-providers/xai) | `grok-4` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
115
116
|
| [xAI Grok](/providers/ai-sdk-providers/xai) | `grok-3` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
116
117
|
| [xAI Grok](/providers/ai-sdk-providers/xai) | `grok-3-mini` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
117
118
|
| [Vercel](/providers/ai-sdk-providers/vercel) | `v0-1.0-md` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
119
|
+
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.6` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
120
|
+
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.6-luna` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
121
|
+
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.6-sol` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
122
|
+
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.6-terra` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
118
123
|
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.5` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
119
124
|
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.4-pro` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
120
125
|
| [OpenAI](/providers/ai-sdk-providers/openai) | `gpt-5.4` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
|
|
@@ -37,7 +37,7 @@ const uint8Array = video.uint8Array; // Uint8Array video data
|
|
|
37
37
|
The aspect ratio is specified as a string in the format `{width}:{height}`.
|
|
38
38
|
Models only support a few aspect ratios, and the supported aspect ratios are different for each model and provider.
|
|
39
39
|
|
|
40
|
-
```tsx highlight={"
|
|
40
|
+
```tsx highlight={"6"}
|
|
41
41
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
42
42
|
__PROVIDER_IMPORT__;
|
|
43
43
|
|
|
@@ -53,7 +53,7 @@ const { video } = await generateVideo({
|
|
|
53
53
|
The resolution is specified as a string in the format `{width}x{height}`.
|
|
54
54
|
Models only support specific resolutions, and the supported resolutions are different for each model and provider.
|
|
55
55
|
|
|
56
|
-
```tsx highlight={"
|
|
56
|
+
```tsx highlight={"6"}
|
|
57
57
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
58
58
|
__PROVIDER_IMPORT__;
|
|
59
59
|
|
|
@@ -68,7 +68,7 @@ const { video } = await generateVideo({
|
|
|
68
68
|
|
|
69
69
|
Some video models support specifying the duration of the generated video in seconds.
|
|
70
70
|
|
|
71
|
-
```tsx highlight={"
|
|
71
|
+
```tsx highlight={"6"}
|
|
72
72
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
73
73
|
__PROVIDER_IMPORT__;
|
|
74
74
|
|
|
@@ -83,7 +83,7 @@ const { video } = await generateVideo({
|
|
|
83
83
|
|
|
84
84
|
Some video models allow you to specify the frames per second for the generated video.
|
|
85
85
|
|
|
86
|
-
```tsx highlight={"
|
|
86
|
+
```tsx highlight={"6"}
|
|
87
87
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
88
88
|
__PROVIDER_IMPORT__;
|
|
89
89
|
|
|
@@ -98,7 +98,7 @@ const { video } = await generateVideo({
|
|
|
98
98
|
|
|
99
99
|
Some video models can generate audio alongside the video. Use the `generateAudio` option to control this:
|
|
100
100
|
|
|
101
|
-
```tsx highlight={"
|
|
101
|
+
```tsx highlight={"6"}
|
|
102
102
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
103
103
|
__PROVIDER_IMPORT__;
|
|
104
104
|
|
|
@@ -113,7 +113,7 @@ const { video } = await generateVideo({
|
|
|
113
113
|
|
|
114
114
|
`experimental_generateVideo` supports generating multiple videos at once:
|
|
115
115
|
|
|
116
|
-
```tsx highlight={"
|
|
116
|
+
```tsx highlight={"6"}
|
|
117
117
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
118
118
|
__PROVIDER_IMPORT__;
|
|
119
119
|
|
|
@@ -146,7 +146,7 @@ const { videos } = await generateVideo({
|
|
|
146
146
|
|
|
147
147
|
Some video models support generating videos from an input image. You can provide an image using the prompt object:
|
|
148
148
|
|
|
149
|
-
```tsx highlight={"7
|
|
149
|
+
```tsx highlight={"5-7"}
|
|
150
150
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
151
151
|
__PROVIDER_IMPORT__;
|
|
152
152
|
|
|
@@ -177,7 +177,7 @@ Some video models support first-last-frame generation, where you provide the
|
|
|
177
177
|
starting and/or ending frames of the video. Use the `frameImages` option to pass
|
|
178
178
|
role-tagged images in a provider-agnostic way:
|
|
179
179
|
|
|
180
|
-
```tsx highlight={"
|
|
180
|
+
```tsx highlight={"4-13"}
|
|
181
181
|
const { video } = await generateVideo({
|
|
182
182
|
model: __VIDEO_MODEL__,
|
|
183
183
|
prompt: 'The cat walks across the scene and transforms into a dog by the end',
|
|
@@ -194,13 +194,13 @@ const { video } = await generateVideo({
|
|
|
194
194
|
});
|
|
195
195
|
```
|
|
196
196
|
|
|
197
|
-
### Reference
|
|
197
|
+
### Reference Inputs
|
|
198
198
|
|
|
199
199
|
Some video models support reference-to-video generation, where you provide one or
|
|
200
|
-
more reference images that the model incorporates into the generated video. Use the `inputReferences` option to pass
|
|
201
|
-
these
|
|
200
|
+
more reference images or videos that the model incorporates into the generated video. Use the `inputReferences` option to pass
|
|
201
|
+
these inputs in a provider-agnostic way:
|
|
202
202
|
|
|
203
|
-
```tsx highlight={"
|
|
203
|
+
```tsx highlight={"4-7"}
|
|
204
204
|
const { video } = await generateVideo({
|
|
205
205
|
model: __VIDEO_MODEL__,
|
|
206
206
|
prompt: 'The two characters meet in a bustling market',
|
|
@@ -211,12 +211,31 @@ const { video } = await generateVideo({
|
|
|
211
211
|
});
|
|
212
212
|
```
|
|
213
213
|
|
|
214
|
+
For URL-based video references, use the object form with an explicit `mediaType`:
|
|
215
|
+
|
|
216
|
+
```tsx highlight={"4-9"}
|
|
217
|
+
const { video } = await generateVideo({
|
|
218
|
+
model: __VIDEO_MODEL__,
|
|
219
|
+
prompt: 'Match the motion in the reference clip',
|
|
220
|
+
inputReferences: [
|
|
221
|
+
{
|
|
222
|
+
data: 'https://example.com/reference.mp4',
|
|
223
|
+
mediaType: 'video/mp4',
|
|
224
|
+
},
|
|
225
|
+
],
|
|
226
|
+
});
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Providers route each reference by its media type (image vs. video) and emit a
|
|
230
|
+
warning when a reference kind is unsupported (for example, providers that accept
|
|
231
|
+
only image references warn and ignore a video reference).
|
|
232
|
+
|
|
214
233
|
### Providing a Seed
|
|
215
234
|
|
|
216
235
|
You can provide a seed to the `experimental_generateVideo` function to control the output of the video generation process.
|
|
217
236
|
If supported by the model, the same seed will always produce the same video.
|
|
218
237
|
|
|
219
|
-
```tsx highlight={"
|
|
238
|
+
```tsx highlight={"6"}
|
|
220
239
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
221
240
|
__PROVIDER_IMPORT__;
|
|
222
241
|
|
|
@@ -254,7 +273,7 @@ const { video } = await generateVideo({
|
|
|
254
273
|
type [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal)
|
|
255
274
|
that you can use to abort the video generation process or set a timeout.
|
|
256
275
|
|
|
257
|
-
```ts highlight={"
|
|
276
|
+
```ts highlight={"6"}
|
|
258
277
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
259
278
|
__PROVIDER_IMPORT__;
|
|
260
279
|
|
|
@@ -276,7 +295,7 @@ Video generation is an asynchronous process that can take several minutes to com
|
|
|
276
295
|
|
|
277
296
|
You can configure the polling timeout using provider-specific options. Each provider exports a type for its options that you can use with `satisfies` for type safety:
|
|
278
297
|
|
|
279
|
-
```tsx highlight={"
|
|
298
|
+
```tsx highlight={"9-11"}
|
|
280
299
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
281
300
|
import { fal, type FalVideoModelOptions } from '@ai-sdk/fal';
|
|
282
301
|
|
|
@@ -303,7 +322,7 @@ const { video } = await generateVideo({
|
|
|
303
322
|
`experimental_generateVideo` accepts an optional `headers` parameter of type `Record<string, string>`
|
|
304
323
|
that you can use to add custom headers to the video generation request.
|
|
305
324
|
|
|
306
|
-
```ts highlight={"
|
|
325
|
+
```ts highlight={"6"}
|
|
307
326
|
import { experimental_generateVideo as generateVideo } from 'ai';
|
|
308
327
|
__PROVIDER_IMPORT__;
|
|
309
328
|
|
|
@@ -118,9 +118,10 @@ console.log(videos);
|
|
|
118
118
|
},
|
|
119
119
|
{
|
|
120
120
|
name: 'inputReferences',
|
|
121
|
-
type: 'Array<DataContent>',
|
|
121
|
+
type: 'Array<DataContent | { data: DataContent; mediaType?: string }>',
|
|
122
122
|
isOptional: true,
|
|
123
|
-
description:
|
|
123
|
+
description:
|
|
124
|
+
'Reference image or video inputs for reference-to-video generation. Use the object form with an explicit mediaType for URL-based video references. Providers route each reference by its media type and warn when a reference kind is unsupported.',
|
|
124
125
|
},
|
|
125
126
|
{
|
|
126
127
|
name: 'generateAudio',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai",
|
|
3
|
-
"version": "6.0.
|
|
3
|
+
"version": "6.0.222",
|
|
4
4
|
"description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -45,9 +45,9 @@
|
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"@opentelemetry/api": "^1.9.0",
|
|
48
|
-
"@ai-sdk/gateway": "3.0.
|
|
49
|
-
"@ai-sdk/provider": "3.0.
|
|
50
|
-
"@ai-sdk/provider-utils": "4.0.
|
|
48
|
+
"@ai-sdk/gateway": "3.0.146",
|
|
49
|
+
"@ai-sdk/provider": "3.0.14",
|
|
50
|
+
"@ai-sdk/provider-utils": "4.0.38"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@edge-runtime/vm": "^5.0.0",
|
|
@@ -52,6 +52,7 @@ import { asArray } from '../util/as-array';
|
|
|
52
52
|
import type { DownloadFunction } from '../util/download/download-function';
|
|
53
53
|
import { mergeObjects } from '../util/merge-objects';
|
|
54
54
|
import { prepareRetries } from '../util/prepare-retries';
|
|
55
|
+
import { setAbortTimeout } from '../util/set-abort-timeout';
|
|
55
56
|
import { VERSION } from '../version';
|
|
56
57
|
import type {
|
|
57
58
|
OnFinishEvent,
|
|
@@ -684,10 +685,11 @@ export async function generateText<
|
|
|
684
685
|
|
|
685
686
|
do {
|
|
686
687
|
// Set up step timeout if configured
|
|
687
|
-
const stepTimeoutId =
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
688
|
+
const stepTimeoutId = setAbortTimeout({
|
|
689
|
+
abortController: stepAbortController,
|
|
690
|
+
label: 'Step',
|
|
691
|
+
timeoutMs: stepTimeoutMs,
|
|
692
|
+
});
|
|
691
693
|
|
|
692
694
|
try {
|
|
693
695
|
const stepInputMessages = [...initialMessages, ...responseMessages];
|
|
@@ -278,6 +278,7 @@ export const array = <ELEMENT>({
|
|
|
278
278
|
});
|
|
279
279
|
}
|
|
280
280
|
|
|
281
|
+
const validatedElements: Array<ELEMENT> = [];
|
|
281
282
|
for (const element of outerValue.elements) {
|
|
282
283
|
const validationResult = await safeValidateTypes({
|
|
283
284
|
value: element,
|
|
@@ -294,9 +295,11 @@ export const array = <ELEMENT>({
|
|
|
294
295
|
finishReason: context.finishReason,
|
|
295
296
|
});
|
|
296
297
|
}
|
|
298
|
+
|
|
299
|
+
validatedElements.push(validationResult.value);
|
|
297
300
|
}
|
|
298
301
|
|
|
299
|
-
return
|
|
302
|
+
return validatedElements;
|
|
300
303
|
},
|
|
301
304
|
|
|
302
305
|
async parsePartialOutput({ text }: { text: string }) {
|
|
@@ -171,10 +171,8 @@ export function runToolsTransformation<TOOLS extends ToolSet>({
|
|
|
171
171
|
// keep track of outstanding tool results for stream closing:
|
|
172
172
|
const outstandingToolResults = new Set<string>();
|
|
173
173
|
|
|
174
|
-
// keep track of tool inputs for provider-side tool results
|
|
175
|
-
const toolInputs = new Map<string, unknown>();
|
|
176
|
-
|
|
177
174
|
// keep track of parsed tool calls so provider-emitted approval requests can reference them
|
|
175
|
+
// keep track of tool inputs for provider-side tool results
|
|
178
176
|
const toolCallsByToolCallId = new Map<string, TypedToolCall<TOOLS>>();
|
|
179
177
|
|
|
180
178
|
let canClose = false;
|
|
@@ -349,8 +347,6 @@ export function runToolsTransformation<TOOLS extends ToolSet>({
|
|
|
349
347
|
break;
|
|
350
348
|
}
|
|
351
349
|
|
|
352
|
-
toolInputs.set(toolCall.toolCallId, toolCall.input);
|
|
353
|
-
|
|
354
350
|
// Only execute tools that are not provider-executed:
|
|
355
351
|
if (tool.execute != null && toolCall.providerExecuted !== true) {
|
|
356
352
|
const toolExecutionId = generateId(); // use our own id to guarantee uniqueness
|
|
@@ -405,7 +401,7 @@ export function runToolsTransformation<TOOLS extends ToolSet>({
|
|
|
405
401
|
type: 'tool-error',
|
|
406
402
|
toolCallId: chunk.toolCallId,
|
|
407
403
|
toolName,
|
|
408
|
-
input:
|
|
404
|
+
input: toolCall?.input,
|
|
409
405
|
providerExecuted: true,
|
|
410
406
|
error: chunk.result,
|
|
411
407
|
dynamic: chunk.dynamic,
|
|
@@ -421,7 +417,7 @@ export function runToolsTransformation<TOOLS extends ToolSet>({
|
|
|
421
417
|
type: 'tool-result',
|
|
422
418
|
toolCallId: chunk.toolCallId,
|
|
423
419
|
toolName,
|
|
424
|
-
input:
|
|
420
|
+
input: toolCall?.input,
|
|
425
421
|
output: chunk.result,
|
|
426
422
|
providerExecuted: true,
|
|
427
423
|
dynamic: chunk.dynamic,
|
|
@@ -82,6 +82,7 @@ import { mergeAbortSignals } from '../util/merge-abort-signals';
|
|
|
82
82
|
import { mergeObjects } from '../util/merge-objects';
|
|
83
83
|
import { now as originalNow } from '../util/now';
|
|
84
84
|
import { prepareRetries } from '../util/prepare-retries';
|
|
85
|
+
import { setAbortTimeout } from '../util/set-abort-timeout';
|
|
85
86
|
import { collectToolApprovals } from './collect-tool-approvals';
|
|
86
87
|
import { validateApprovedToolApprovals } from './validate-tool-approvals';
|
|
87
88
|
import type {
|
|
@@ -1567,25 +1568,25 @@ class DefaultStreamTextResult<
|
|
|
1567
1568
|
const includeRawChunks = self.includeRawChunks;
|
|
1568
1569
|
|
|
1569
1570
|
// Set up step timeout if configured
|
|
1570
|
-
const stepTimeoutId =
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1571
|
+
const stepTimeoutId = setAbortTimeout({
|
|
1572
|
+
abortController: stepAbortController,
|
|
1573
|
+
label: 'Step',
|
|
1574
|
+
timeoutMs: stepTimeoutMs,
|
|
1575
|
+
});
|
|
1574
1576
|
|
|
1575
1577
|
// Set up chunk timeout tracking (will be reset on each chunk)
|
|
1576
1578
|
let chunkTimeoutId: ReturnType<typeof setTimeout> | undefined =
|
|
1577
1579
|
undefined;
|
|
1578
1580
|
|
|
1579
1581
|
function resetChunkTimeout() {
|
|
1580
|
-
if (
|
|
1581
|
-
|
|
1582
|
-
clearTimeout(chunkTimeoutId);
|
|
1583
|
-
}
|
|
1584
|
-
chunkTimeoutId = setTimeout(
|
|
1585
|
-
() => chunkAbortController!.abort(),
|
|
1586
|
-
chunkTimeoutMs,
|
|
1587
|
-
);
|
|
1582
|
+
if (chunkTimeoutId != null) {
|
|
1583
|
+
clearTimeout(chunkTimeoutId);
|
|
1588
1584
|
}
|
|
1585
|
+
chunkTimeoutId = setAbortTimeout({
|
|
1586
|
+
abortController: chunkAbortController,
|
|
1587
|
+
label: 'Chunk',
|
|
1588
|
+
timeoutMs: chunkTimeoutMs,
|
|
1589
|
+
});
|
|
1589
1590
|
}
|
|
1590
1591
|
|
|
1591
1592
|
function clearChunkTimeout() {
|
|
@@ -1601,6 +1602,13 @@ class DefaultStreamTextResult<
|
|
|
1601
1602
|
}
|
|
1602
1603
|
}
|
|
1603
1604
|
|
|
1605
|
+
// The step's stream is registered lazily and consumed long after this
|
|
1606
|
+
// function returns, so the step timer must stay armed past setup. When
|
|
1607
|
+
// the merged abort signal fires (any step/chunk/total timeout or caller
|
|
1608
|
+
// abort), drop both step-scoped timers so neither outlives the step.
|
|
1609
|
+
abortSignal?.addEventListener('abort', clearStepTimeout);
|
|
1610
|
+
abortSignal?.addEventListener('abort', clearChunkTimeout);
|
|
1611
|
+
|
|
1604
1612
|
try {
|
|
1605
1613
|
stepFinish = new DelayedPromise<void>();
|
|
1606
1614
|
|
|
@@ -2244,9 +2252,12 @@ class DefaultStreamTextResult<
|
|
|
2244
2252
|
}),
|
|
2245
2253
|
),
|
|
2246
2254
|
);
|
|
2247
|
-
}
|
|
2255
|
+
} catch (error) {
|
|
2256
|
+
// Setup failed before the stream was registered, so neither the
|
|
2257
|
+
// stream's flush nor an abort will clear the timers — clear them here.
|
|
2248
2258
|
clearStepTimeout();
|
|
2249
2259
|
clearChunkTimeout();
|
|
2260
|
+
throw error;
|
|
2250
2261
|
}
|
|
2251
2262
|
}
|
|
2252
2263
|
|
|
@@ -32,7 +32,6 @@ import { prepareRetries } from '../util/prepare-retries';
|
|
|
32
32
|
import { VERSION } from '../version';
|
|
33
33
|
import type { GenerateVideoResult } from './generate-video-result';
|
|
34
34
|
import { splitDataUrl } from '../prompt/split-data-url';
|
|
35
|
-
import { convertDataContentToUint8Array } from '../prompt/data-content';
|
|
36
35
|
|
|
37
36
|
export type GenerateVideoPrompt =
|
|
38
37
|
| string
|
|
@@ -53,7 +52,7 @@ export type GenerateVideoPrompt =
|
|
|
53
52
|
* @param fps - Frames per second for the video.
|
|
54
53
|
* @param seed - Seed for the video generation.
|
|
55
54
|
* @param frameImages - Role-tagged image inputs for image-to-video and first-last-frame generation.
|
|
56
|
-
* @param inputReferences - Reference image inputs for reference-to-video generation.
|
|
55
|
+
* @param inputReferences - Reference image or video inputs for reference-to-video generation.
|
|
57
56
|
* @param generateAudio - Whether the model should generate audio alongside the video.
|
|
58
57
|
* @param providerOptions - Additional provider-specific options that are passed through to the provider
|
|
59
58
|
* as body parameters.
|
|
@@ -145,9 +144,26 @@ export async function experimental_generateVideo({
|
|
|
145
144
|
}>;
|
|
146
145
|
|
|
147
146
|
/**
|
|
148
|
-
* Reference
|
|
147
|
+
* Reference inputs for reference-to-video generation.
|
|
148
|
+
*
|
|
149
|
+
* Each entry may be a plain image/video ({@link DataContent}), or an object
|
|
150
|
+
* form that carries an explicit `mediaType`.
|
|
149
151
|
*/
|
|
150
|
-
inputReferences?: Array<
|
|
152
|
+
inputReferences?: Array<
|
|
153
|
+
| DataContent
|
|
154
|
+
| {
|
|
155
|
+
/**
|
|
156
|
+
* The reference image or video.
|
|
157
|
+
*/
|
|
158
|
+
data: DataContent;
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The media type of the reference (e.g. 'image/png',
|
|
162
|
+
* 'video/mp4').
|
|
163
|
+
*/
|
|
164
|
+
mediaType?: string;
|
|
165
|
+
}
|
|
166
|
+
>;
|
|
151
167
|
|
|
152
168
|
/**
|
|
153
169
|
* Whether the model should generate audio alongside the video.
|
|
@@ -205,16 +221,19 @@ export async function experimental_generateVideo({
|
|
|
205
221
|
|
|
206
222
|
const normalizedFrameImages:
|
|
207
223
|
| Array<Experimental_VideoModelV3FrameImage>
|
|
208
|
-
| undefined = frameImages?.
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
224
|
+
| undefined = frameImages?.flatMap(frame => {
|
|
225
|
+
const normalizedImage = normalizeImageData(frame.image);
|
|
226
|
+
return normalizedImage != null
|
|
227
|
+
? [{ image: normalizedImage, frameType: frame.frameType }]
|
|
228
|
+
: [];
|
|
229
|
+
});
|
|
212
230
|
|
|
213
231
|
const normalizedInputReferences:
|
|
214
232
|
| Array<Experimental_VideoModelV3File>
|
|
215
|
-
| undefined = inputReferences?.
|
|
216
|
-
|
|
217
|
-
|
|
233
|
+
| undefined = inputReferences?.flatMap(reference => {
|
|
234
|
+
const normalized = normalizeReferenceData(reference);
|
|
235
|
+
return normalized != null ? [normalized] : [];
|
|
236
|
+
});
|
|
218
237
|
|
|
219
238
|
const effectiveInputReferences =
|
|
220
239
|
normalizedFrameImages != null && normalizedFrameImages.length > 0
|
|
@@ -430,13 +449,25 @@ function normalizePrompt(promptArg: GenerateVideoPrompt): {
|
|
|
430
449
|
};
|
|
431
450
|
}
|
|
432
451
|
|
|
452
|
+
function detectFileMediaType(
|
|
453
|
+
data: Uint8Array,
|
|
454
|
+
restrictToImages: boolean,
|
|
455
|
+
): string {
|
|
456
|
+
const detected = restrictToImages
|
|
457
|
+
? detectMediaType({ data, signatures: imageMediaTypeSignatures })
|
|
458
|
+
: (detectMediaType({ data, signatures: imageMediaTypeSignatures }) ??
|
|
459
|
+
detectMediaType({ data, signatures: videoMediaTypeSignatures }));
|
|
460
|
+
return detected ?? 'image/png';
|
|
461
|
+
}
|
|
462
|
+
|
|
433
463
|
/**
|
|
434
464
|
* Normalizes a {@link DataContent} image into a {@link Experimental_VideoModelV3File}.
|
|
435
465
|
* Accepts a URL string, a data URL, a base64 string, or binary image data.
|
|
436
466
|
*/
|
|
437
467
|
function normalizeImageData(
|
|
438
468
|
dataContent: DataContent,
|
|
439
|
-
|
|
469
|
+
{ restrictToImages = true }: { restrictToImages?: boolean } = {},
|
|
470
|
+
): Experimental_VideoModelV3File | undefined {
|
|
440
471
|
if (typeof dataContent === 'string') {
|
|
441
472
|
if (
|
|
442
473
|
dataContent.startsWith('http://') ||
|
|
@@ -450,32 +481,73 @@ function normalizeImageData(
|
|
|
450
481
|
|
|
451
482
|
if (dataContent.startsWith('data:')) {
|
|
452
483
|
const { mediaType, base64Content } = splitDataUrl(dataContent);
|
|
484
|
+
const data = convertBase64ToUint8Array(base64Content ?? '');
|
|
453
485
|
return {
|
|
454
486
|
type: 'file',
|
|
455
|
-
mediaType: mediaType ??
|
|
456
|
-
data
|
|
487
|
+
mediaType: mediaType ?? detectFileMediaType(data, restrictToImages),
|
|
488
|
+
data,
|
|
457
489
|
};
|
|
458
490
|
}
|
|
459
491
|
|
|
460
492
|
const bytes = convertBase64ToUint8Array(dataContent);
|
|
461
493
|
return {
|
|
462
494
|
type: 'file',
|
|
463
|
-
mediaType:
|
|
464
|
-
detectMediaType({
|
|
465
|
-
data: bytes,
|
|
466
|
-
signatures: imageMediaTypeSignatures,
|
|
467
|
-
}) ?? 'image/png',
|
|
495
|
+
mediaType: detectFileMediaType(bytes, restrictToImages),
|
|
468
496
|
data: bytes,
|
|
469
497
|
};
|
|
470
498
|
}
|
|
471
499
|
|
|
472
|
-
|
|
500
|
+
if (dataContent instanceof Uint8Array || dataContent instanceof ArrayBuffer) {
|
|
501
|
+
const bytes =
|
|
502
|
+
dataContent instanceof Uint8Array
|
|
503
|
+
? dataContent
|
|
504
|
+
: new Uint8Array(dataContent);
|
|
505
|
+
return {
|
|
506
|
+
type: 'file',
|
|
507
|
+
mediaType: detectFileMediaType(bytes, restrictToImages),
|
|
508
|
+
data: bytes,
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
return undefined;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Normalizes a reference input into a {@link Experimental_VideoModelV3File},
|
|
517
|
+
* accepting either a plain {@link DataContent} or the object form that carries
|
|
518
|
+
* an explicit `mediaType`.
|
|
519
|
+
*/
|
|
520
|
+
function normalizeReferenceData(
|
|
521
|
+
reference:
|
|
522
|
+
| DataContent
|
|
523
|
+
| {
|
|
524
|
+
data: DataContent;
|
|
525
|
+
mediaType?: string;
|
|
526
|
+
},
|
|
527
|
+
): Experimental_VideoModelV3File | undefined {
|
|
528
|
+
const isObjectForm =
|
|
529
|
+
typeof reference === 'object' &&
|
|
530
|
+
reference != null &&
|
|
531
|
+
!(reference instanceof Uint8Array) &&
|
|
532
|
+
!(reference instanceof ArrayBuffer) &&
|
|
533
|
+
'data' in reference;
|
|
534
|
+
|
|
535
|
+
if (!isObjectForm) {
|
|
536
|
+
return normalizeImageData(reference as DataContent, {
|
|
537
|
+
restrictToImages: false,
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
const normalized = normalizeImageData(reference.data, {
|
|
542
|
+
restrictToImages: false,
|
|
543
|
+
});
|
|
544
|
+
if (normalized == null) {
|
|
545
|
+
return normalized;
|
|
546
|
+
}
|
|
547
|
+
|
|
473
548
|
return {
|
|
474
|
-
|
|
475
|
-
mediaType:
|
|
476
|
-
detectMediaType({ data: bytes, signatures: imageMediaTypeSignatures }) ??
|
|
477
|
-
'image/png',
|
|
478
|
-
data: bytes,
|
|
549
|
+
...normalized,
|
|
550
|
+
...(reference.mediaType != null ? { mediaType: reference.mediaType } : {}),
|
|
479
551
|
};
|
|
480
552
|
}
|
|
481
553
|
|
package/src/ui/chat.ts
CHANGED
|
@@ -246,10 +246,12 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
|
|
|
246
246
|
}
|
|
247
247
|
}
|
|
248
248
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
249
|
+
if (content.length > 0) {
|
|
250
|
+
modelMessages.push({
|
|
251
|
+
role: 'assistant',
|
|
252
|
+
content,
|
|
253
|
+
});
|
|
254
|
+
}
|
|
253
255
|
|
|
254
256
|
// check if there are tool invocations with results in the block
|
|
255
257
|
// Include non-provider-executed tools, OR provider-executed tools with approval responses
|
|
@@ -297,7 +299,7 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
|
|
|
297
299
|
type: 'error-text' as const,
|
|
298
300
|
value:
|
|
299
301
|
toolPart.approval?.reason ??
|
|
300
|
-
'Tool execution denied.',
|
|
302
|
+
'Tool call execution denied.',
|
|
301
303
|
},
|
|
302
304
|
...(toolPart.callProviderMetadata != null
|
|
303
305
|
? { providerOptions: toolPart.callProviderMetadata }
|
package/src/ui/index.ts
CHANGED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schedules a timeout that aborts the given controller with a `TimeoutError`
|
|
3
|
+
* `DOMException`, matching the reason produced by `AbortSignal.timeout(ms)`.
|
|
4
|
+
*
|
|
5
|
+
* @param abortController - The controller to abort when the timeout elapses.
|
|
6
|
+
* If undefined, no timeout is scheduled.
|
|
7
|
+
* @param label - Human-readable label included in the error message
|
|
8
|
+
* (e.g. "Step", "Chunk").
|
|
9
|
+
* @param timeoutMs - Duration in milliseconds before the controller is aborted.
|
|
10
|
+
* If undefined, no timeout is scheduled.
|
|
11
|
+
* @returns The timeout id (suitable for passing to `clearTimeout`), or
|
|
12
|
+
* `undefined` if no timeout was scheduled.
|
|
13
|
+
*/
|
|
14
|
+
export function setAbortTimeout({
|
|
15
|
+
abortController,
|
|
16
|
+
label,
|
|
17
|
+
timeoutMs,
|
|
18
|
+
}: {
|
|
19
|
+
abortController: AbortController | undefined;
|
|
20
|
+
label: string;
|
|
21
|
+
timeoutMs: number | undefined;
|
|
22
|
+
}): ReturnType<typeof setTimeout> | undefined {
|
|
23
|
+
if (abortController == null || timeoutMs == null) {
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
return setTimeout(
|
|
28
|
+
() =>
|
|
29
|
+
abortController.abort(
|
|
30
|
+
new DOMException(
|
|
31
|
+
`${label} timeout of ${timeoutMs}ms exceeded`,
|
|
32
|
+
'TimeoutError',
|
|
33
|
+
),
|
|
34
|
+
),
|
|
35
|
+
timeoutMs,
|
|
36
|
+
);
|
|
37
|
+
}
|