nexus-ai-pro 2.16.0 → 2.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/API_STABILITY.md +22 -0
  2. package/CHANGELOG.md +54 -1
  3. package/README.md +6 -4
  4. package/dist/agent/capabilities.d.ts +3 -0
  5. package/dist/mcp/client.d.ts +62 -3
  6. package/dist/mcp/client.js +154 -16
  7. package/dist/mcp/index.d.ts +2 -2
  8. package/dist/mcp/index.js +2 -2
  9. package/dist/mcp/server.d.ts +12 -0
  10. package/dist/mcp/server.js +24 -5
  11. package/dist/protocols/acp-editor.d.ts +16 -2
  12. package/dist/protocols/acp-editor.js +21 -9
  13. package/dist/protocols/acp.d.ts +8 -0
  14. package/dist/protocols/acp.js +1 -0
  15. package/dist/version.d.ts +1 -1
  16. package/dist/version.js +1 -1
  17. package/dist/video/google.d.ts +42 -11
  18. package/dist/video/google.js +249 -20
  19. package/dist/video/index.d.ts +49 -6
  20. package/dist/video/index.js +91 -15
  21. package/dist/video/mock.js +10 -1
  22. package/dist/video/runway.d.ts +8 -1
  23. package/dist/video/runway.js +68 -2
  24. package/dist-cjs/agent/capabilities.d.ts +3 -0
  25. package/dist-cjs/mcp/client.d.ts +62 -3
  26. package/dist-cjs/mcp/client.js +156 -16
  27. package/dist-cjs/mcp/index.d.ts +2 -2
  28. package/dist-cjs/mcp/index.js +4 -1
  29. package/dist-cjs/mcp/server.d.ts +12 -0
  30. package/dist-cjs/mcp/server.js +25 -5
  31. package/dist-cjs/protocols/acp-editor.d.ts +16 -2
  32. package/dist-cjs/protocols/acp-editor.js +20 -8
  33. package/dist-cjs/protocols/acp.d.ts +8 -0
  34. package/dist-cjs/protocols/acp.js +1 -0
  35. package/dist-cjs/version.d.ts +1 -1
  36. package/dist-cjs/version.js +1 -1
  37. package/dist-cjs/video/google.d.ts +42 -11
  38. package/dist-cjs/video/google.js +248 -19
  39. package/dist-cjs/video/index.d.ts +49 -6
  40. package/dist-cjs/video/index.js +91 -15
  41. package/dist-cjs/video/mock.js +10 -1
  42. package/dist-cjs/video/runway.d.ts +8 -1
  43. package/dist-cjs/video/runway.js +68 -2
  44. package/package.json +1 -1
@@ -11,15 +11,25 @@ export interface GoogleVideoProviderConfig {
11
11
  accessToken?: string | (() => string | Promise<string>);
12
12
  /** API base URL. Defaults to the Gemini API, `https://generativelanguage.googleapis.com/v1beta`. */
13
13
  baseUrl?: string;
14
- /** Vertex AI only: where Veo writes each video, as `gs://bucket/prefix`. Without it Vertex returns the bytes inline. */
14
+ /**
15
+ * Vertex AI only: where Veo writes each video, as `gs://bucket/prefix`. Without it Vertex returns the
16
+ * bytes inline, and a job cannot be extended from what it made.
17
+ */
15
18
  storageUri?: string;
16
- /** The model a request naming none uses. Defaults to `veo-3.1-generate-preview`. */
19
+ /**
20
+ * The model a request naming none uses. Defaults to `gemini-omni-1.1-flash` on the Gemini API and
21
+ * `veo-3.1-generate-001` on Vertex AI.
22
+ */
17
23
  defaultModel?: string;
18
- /** The models served. Defaults to the Veo 3.1 family; each takes what Veo 3.1 takes unless `byModel` says otherwise. */
24
+ /**
25
+ * The models served. Defaults to Gemini Omni Flash on the Gemini API, where the Veo 3.1 previews shut
26
+ * down on 2026-10-22, and to Veo 3.1 on Vertex AI. A model named `gemini-omni…` is reached through the
27
+ * Interactions API, any other through Veo's protocol.
28
+ */
19
29
  models?: readonly string[];
20
- /** Where a model differs from Veo 3.1. */
30
+ /** Where a model differs from what the adapter knows of it. */
21
31
  byModel?: Record<string, Partial<VideoModelCapabilities>>;
22
- /** `dont_allow`, `allow_adult`, or `allow_all`, sent as `personGeneration`. Some regions allow only `allow_adult`. */
32
+ /** `dont_allow`, `allow_adult`, or `allow_all`, sent to Veo as `personGeneration`. Some regions allow only `allow_adult`. */
23
33
  personGeneration?: string;
24
34
  /** Headers added to every request. */
25
35
  defaultHeaders?: Record<string, string>;
@@ -27,9 +37,15 @@ export interface GoogleVideoProviderConfig {
27
37
  fetch?: typeof globalThis.fetch;
28
38
  }
29
39
  /**
30
- * Veo behind the video contract. A job is a long-running operation: its name is the ref, any worker
31
- * polls it, and the video is read from the URI the Gemini API returns, from Cloud Storage, or from the
32
- * bytes Vertex returns inline. The Gemini API keeps a video for two days.
40
+ * Google's video models behind the video contract.
41
+ *
42
+ * Gemini Omni Flash runs as a background interaction: its id is the ref, any worker reads it, and its
43
+ * video comes back with it, inline or by a Files API URI. It extends or edits a video given to it, or
44
+ * continues an interaction of its own by its id, with nothing uploaded.
45
+ *
46
+ * Veo runs as a long-running operation: its name is the ref, any worker polls it, and the video is read
47
+ * from the URI the Gemini API returns, from Cloud Storage, or from the bytes Vertex returns inline. On
48
+ * Vertex it extends a video in Cloud Storage by seven seconds, or a job of its own that wrote there.
33
49
  */
34
50
  export declare class GoogleVideoProvider implements VideoProvider {
35
51
  private readonly config;
@@ -38,14 +54,29 @@ export declare class GoogleVideoProvider implements VideoProvider {
38
54
  private readonly baseUrl;
39
55
  private readonly fetchImplementation;
40
56
  private readonly vertex;
57
+ /** Interactions read as finished, so the download right after reads no second copy of the video. */
58
+ private readonly finished;
41
59
  constructor(config: GoogleVideoProviderConfig);
42
- /** Starts a long-running prediction. */
60
+ /** What one model takes: Omni's, Veo 3.1's, or Veo's on Vertex, which extends videos too. */
61
+ private capabilitiesFor;
62
+ /** Starts an interaction for Omni, or a long-running prediction for Veo. */
43
63
  submit(request: VideoProviderRequest, context: VideoCallContext): Promise<VideoJobRef>;
44
- /** Reads the operation. */
64
+ /** Reads the interaction or the operation. */
45
65
  poll(ref: VideoJobRef, context: VideoCallContext): Promise<VideoJobState>;
46
- /** Streams one video: from its URI, from Cloud Storage, or from the bytes Vertex returned. */
66
+ /** Streams one video: from its URI, from Cloud Storage, or from the bytes returned with the job. */
47
67
  download(ref: VideoJobRef, index: number, context: VideoCallContext): Promise<VideoContent>;
68
+ /** Cancels a running interaction. A Veo operation runs to its end. */
69
+ cancel(ref: VideoJobRef, context: VideoCallContext): Promise<void>;
70
+ /** Deletes a finished interaction, and its video, from Google. */
71
+ remove(ref: VideoJobRef, context: VideoCallContext): Promise<void>;
72
+ private startInteraction;
73
+ private pollInteraction;
74
+ private interaction;
75
+ /** Keeps the last few finished interactions, whose video the download reads. */
76
+ private remember;
48
77
  private operation;
78
+ /** The video Veo extends: one in Cloud Storage, or one a job of its own wrote there. */
79
+ private veoVideo;
49
80
  private image;
50
81
  private call;
51
82
  private send;
@@ -1,13 +1,9 @@
1
- import { encodeBase64, decodeBase64 } from '../utils/base64.js';
1
+ import { decodeBase64, encodeBase64 } from '../utils/base64.js';
2
2
  import { VideoError } from './errors.js';
3
3
  const GEMINI_API_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta';
4
4
  const STORAGE_BASE_URL = 'https://storage.googleapis.com/storage/v1';
5
- const DEFAULT_MODEL = 'veo-3.1-generate-preview';
6
- const VEO_MODELS = [
7
- 'veo-3.1-generate-preview',
8
- 'veo-3.1-fast-generate-preview',
9
- 'veo-3.1-lite-generate-preview',
10
- ];
5
+ const OMNI_MODEL = 'gemini-omni-1.1-flash';
6
+ const VERTEX_MODELS = ['veo-3.1-generate-001', 'veo-3.1-fast-generate-001', 'veo-3.1-lite-generate-001'];
11
7
  const VEO_31 = {
12
8
  inputs: ['image', 'last-frame', 'references'],
13
9
  durations: [4, 6, 8],
@@ -20,12 +16,28 @@ const VEO_31 = {
20
16
  supportsSeed: true,
21
17
  supportsNegativePrompt: true,
22
18
  };
19
+ const OMNI_LENGTHS = [3, 4, 5, 6, 7, 8, 9, 10];
20
+ const OMNI = {
21
+ inputs: ['image', 'references', 'extend', 'edit'],
22
+ durations: OMNI_LENGTHS,
23
+ defaultDurationSeconds: 8,
24
+ extendDurations: OMNI_LENGTHS,
25
+ aspectRatios: ['16:9', '9:16'],
26
+ resolutions: ['360p', '720p', '1080p', '4k'],
27
+ audio: 'always',
28
+ maxCount: 1,
29
+ supportsSeed: false,
30
+ supportsNegativePrompt: false,
31
+ continuesJobs: true,
32
+ };
33
+ const isOmni = (model) => model.startsWith('gemini-omni');
23
34
  export class GoogleVideoProvider {
24
35
  config;
25
36
  info;
26
37
  baseUrl;
27
38
  fetchImplementation;
28
39
  vertex;
40
+ finished = new Map();
29
41
  constructor(config) {
30
42
  this.config = config;
31
43
  if (!config.apiKey?.trim() && !config.accessToken) {
@@ -38,26 +50,50 @@ export class GoogleVideoProvider {
38
50
  this.fetchImplementation = fetchImplementation;
39
51
  this.vertex = config.accessToken !== undefined;
40
52
  this.baseUrl = (config.baseUrl || GEMINI_API_BASE_URL).replace(/\/+$/, '');
41
- const models = config.models ?? VEO_MODELS;
53
+ const models = config.models ?? (this.vertex ? VERTEX_MODELS : [OMNI_MODEL]);
42
54
  const byModel = {};
43
- if (models.includes('veo-3.1-lite-generate-preview')) {
44
- byModel['veo-3.1-lite-generate-preview'] = { resolutions: ['720p', '1080p'] };
45
- }
55
+ for (const model of models)
56
+ byModel[model] = this.capabilitiesFor(model);
57
+ const defaultModel = config.defaultModel ?? models[0];
58
+ const omniOnly = models.every(isOmni);
46
59
  this.info = {
47
60
  name: 'google',
48
61
  capabilities: {
49
- inputs: VEO_31.inputs ?? [],
50
- ...VEO_31,
62
+ ...(defaultModel ? byModel[defaultModel] : VEO_31),
63
+ inputs: (defaultModel ? byModel[defaultModel]?.inputs : undefined) ?? VEO_31.inputs,
51
64
  models,
52
- defaultModel: config.defaultModel ?? (models.includes(DEFAULT_MODEL) ? DEFAULT_MODEL : models[0]),
53
- byModel: { ...byModel, ...config.byModel },
54
- inputLocationKinds: ['bytes'],
65
+ ...(defaultModel ? { defaultModel } : {}),
66
+ byModel: { ...byModel, ...mergeByModel(byModel, config.byModel) },
67
+ inputLocationKinds: this.vertex || models.some(isOmni) ? ['bytes', 'url'] : ['bytes'],
55
68
  inputMimeTypes: ['image/png', 'image/jpeg', 'image/webp'],
56
- retentionHours: this.vertex ? undefined : 48,
69
+ inputVideoMimeTypes: ['video/mp4'],
70
+ supportsCancel: omniOnly,
71
+ supportsRemove: omniOnly,
72
+ retentionHours: this.vertex ? undefined : omniOnly ? 24 : 48,
57
73
  },
58
74
  };
59
75
  }
76
+ capabilitiesFor(model) {
77
+ if (isOmni(model))
78
+ return OMNI;
79
+ const veo = { ...VEO_31 };
80
+ if (model.includes('lite'))
81
+ veo.resolutions = ['720p', '1080p'];
82
+ if (this.vertex) {
83
+ if (model.includes('fast'))
84
+ veo.resolutions = ['720p', '1080p'];
85
+ if (model.includes('lite'))
86
+ veo.inputs = ['image', 'last-frame'];
87
+ veo.inputs = [...veo.inputs, 'extend'];
88
+ veo.extendDurations = [7];
89
+ veo.maxCount = 4;
90
+ veo.continuesJobs = this.config.storageUri !== undefined;
91
+ }
92
+ return veo;
93
+ }
60
94
  async submit(request, context) {
95
+ if (isOmni(request.model))
96
+ return this.startInteraction(request, context);
61
97
  const instance = { prompt: request.prompt };
62
98
  if (request.image)
63
99
  instance.image = this.image(request.image);
@@ -69,6 +105,8 @@ export class GoogleVideoProvider {
69
105
  referenceType: typeof reference.metadata?.referenceType === 'string' ? reference.metadata.referenceType : 'asset',
70
106
  }));
71
107
  }
108
+ if (request.video)
109
+ instance.video = await this.veoVideo(request, context);
72
110
  const parameters = {};
73
111
  if (request.aspectRatio)
74
112
  parameters.aspectRatio = request.aspectRatio;
@@ -97,6 +135,8 @@ export class GoogleVideoProvider {
97
135
  return { provider: this.info.name, id: operation.name, model: request.model, createdAt: new Date().toISOString() };
98
136
  }
99
137
  async poll(ref, context) {
138
+ if (isInteraction(ref))
139
+ return this.pollInteraction(ref, context);
100
140
  const operation = await this.operation(ref, context);
101
141
  if (!operation.done)
102
142
  return { status: 'running', raw: operation };
@@ -129,7 +169,9 @@ export class GoogleVideoProvider {
129
169
  return { status: 'completed', outputs: samples.length, ...(filtered ? { filtered } : {}), raw: operation };
130
170
  }
131
171
  async download(ref, index, context) {
132
- const sample = samplesOf(await this.operation(ref, context))[index];
172
+ const sample = isInteraction(ref)
173
+ ? videosOf(this.finished.get(ref.id) ?? (await this.interaction(ref.id, context)))[index]
174
+ : samplesOf(await this.operation(ref, context))[index];
133
175
  if (!sample)
134
176
  throw new VideoError(`Job ${ref.id} has no video ${index}`, 'VIDEO_INVALID');
135
177
  if (sample.base64 !== undefined) {
@@ -143,7 +185,7 @@ export class GoogleVideoProvider {
143
185
  return { stream, mimeType: sample.mimeType, byteLength: data.byteLength };
144
186
  }
145
187
  const uri = sample.uri;
146
- const url = uri.startsWith('gs://') ? storageUrl(uri) : uri;
188
+ const url = uri.startsWith('gs://') ? storageUrl(uri) : /^https?:\/\//.test(uri) ? uri : `${this.baseUrl}/${uri}`;
147
189
  const response = await this.send(url, { method: 'GET', redirect: 'follow' }, context);
148
190
  if (!response.ok || !response.body) {
149
191
  throw await failure(response, `Google video download for job ${ref.id} failed`);
@@ -155,15 +197,156 @@ export class GoogleVideoProvider {
155
197
  ...(Number.isSafeInteger(length) && length > 0 ? { byteLength: length } : {}),
156
198
  };
157
199
  }
200
+ async cancel(ref, context) {
201
+ if (!isInteraction(ref))
202
+ throw new VideoError('A Veo operation cannot be cancelled here', 'VIDEO_UNSUPPORTED');
203
+ await this.call(`${this.baseUrl}/interactions/${encodeURIComponent(ref.id)}/cancel`, { method: 'POST' }, context);
204
+ }
205
+ async remove(ref, context) {
206
+ if (!isInteraction(ref))
207
+ throw new VideoError('A Veo video expires on its own', 'VIDEO_UNSUPPORTED');
208
+ this.finished.delete(ref.id);
209
+ await this.call(`${this.baseUrl}/interactions/${encodeURIComponent(ref.id)}`, { method: 'DELETE' }, context);
210
+ }
211
+ async startInteraction(request, context) {
212
+ if (this.vertex) {
213
+ throw new VideoError('Gemini Omni Flash is reached through the Gemini API, not Vertex AI', 'VIDEO_UNSUPPORTED');
214
+ }
215
+ const parts = [];
216
+ if (request.video?.input)
217
+ parts.push(media('video', request.video.input));
218
+ if (request.image)
219
+ parts.push(media('image', request.image));
220
+ for (const reference of request.references ?? [])
221
+ parts.push(media('image', reference));
222
+ if (request.prompt.trim())
223
+ parts.push({ type: 'text', text: request.prompt });
224
+ const format = { type: 'video' };
225
+ if (request.aspectRatio)
226
+ format.aspect_ratio = request.aspectRatio;
227
+ if (request.resolution)
228
+ format.resolution = request.resolution;
229
+ if (request.durationSeconds !== undefined)
230
+ format.duration = `${request.durationSeconds}s`;
231
+ const body = {
232
+ model: request.model,
233
+ input: parts.length === 1 && parts[0]?.type === 'text' ? request.prompt : parts,
234
+ response_format: format,
235
+ background: true,
236
+ store: true,
237
+ };
238
+ const from = request.video?.from;
239
+ if (from) {
240
+ if (!isInteraction(from.ref)) {
241
+ throw new VideoError('Gemini Omni Flash continues only its own interactions; give a Veo video as input', 'VIDEO_UNSUPPORTED');
242
+ }
243
+ if ((from.index ?? 0) !== 0)
244
+ throw new VideoError('An interaction makes one video, at index 0', 'VIDEO_INVALID');
245
+ body.previous_interaction_id = from.ref.id;
246
+ }
247
+ const task = request.video
248
+ ? request.video.mode
249
+ : request.references?.length
250
+ ? 'reference_to_video'
251
+ : request.image
252
+ ? 'image_to_video'
253
+ : undefined;
254
+ if (task)
255
+ body.generation_config = { video_config: { task } };
256
+ Object.assign(body, request.providerOptions);
257
+ const interaction = await this.call(`${this.baseUrl}/interactions`, { method: 'POST', body: JSON.stringify(body) }, context);
258
+ if (!interaction.id)
259
+ throw new VideoError('Google returned no interaction id', 'VIDEO_PROVIDER');
260
+ return {
261
+ provider: this.info.name,
262
+ id: interaction.id,
263
+ model: request.model,
264
+ createdAt: new Date().toISOString(),
265
+ metadata: { google: { api: 'interactions' } },
266
+ };
267
+ }
268
+ async pollInteraction(ref, context) {
269
+ const interaction = await this.interaction(ref.id, context);
270
+ switch (interaction.status) {
271
+ case 'queued':
272
+ return { status: 'queued', raw: summary(interaction) };
273
+ case 'in_progress':
274
+ case 'requires_action':
275
+ return { status: 'running', raw: summary(interaction) };
276
+ case 'cancelled':
277
+ return { status: 'cancelled', raw: summary(interaction) };
278
+ case 'completed': {
279
+ const videos = videosOf(interaction);
280
+ if (videos.length === 0) {
281
+ return {
282
+ status: 'failed',
283
+ error: { message: errorOf(interaction)?.message ?? 'the interaction made no video', retryable: false },
284
+ raw: summary(interaction),
285
+ };
286
+ }
287
+ this.remember(ref.id, interaction);
288
+ return { status: 'completed', outputs: videos.length, raw: summary(interaction) };
289
+ }
290
+ default: {
291
+ const error = errorOf(interaction);
292
+ const code = error?.status ?? (error?.code === undefined ? interaction.status : String(error.code));
293
+ return {
294
+ status: 'failed',
295
+ error: {
296
+ message: error?.message ?? `the interaction ended ${interaction.status ?? 'without a status'}`,
297
+ ...(code ? { code } : {}),
298
+ retryable: [13, 14, 4].includes(error?.code ?? -1),
299
+ ...(/SAFETY|PROHIBITED|BLOCK/i.test(`${error?.status ?? ''} ${error?.message ?? ''}`)
300
+ ? { safety: true }
301
+ : {}),
302
+ },
303
+ raw: summary(interaction),
304
+ };
305
+ }
306
+ }
307
+ }
308
+ interaction(id, context) {
309
+ return this.call(`${this.baseUrl}/interactions/${encodeURIComponent(id)}`, { method: 'GET' }, context);
310
+ }
311
+ remember(id, interaction) {
312
+ this.finished.delete(id);
313
+ this.finished.set(id, interaction);
314
+ while (this.finished.size > 4)
315
+ this.finished.delete(this.finished.keys().next().value);
316
+ }
158
317
  operation(ref, context) {
159
318
  if (this.vertex) {
160
319
  return this.call(`${this.baseUrl}/models/${encodeURIComponent(ref.model)}:fetchPredictOperation`, { method: 'POST', body: JSON.stringify({ operationName: ref.id }) }, context);
161
320
  }
162
321
  return this.call(`${this.baseUrl}/${ref.id}`, { method: 'GET' }, context);
163
322
  }
323
+ async veoVideo(request, context) {
324
+ const video = request.video;
325
+ if (!this.vertex || video?.mode !== 'extend') {
326
+ throw new VideoError('Veo extends videos on Vertex AI, and edits none', 'VIDEO_UNSUPPORTED');
327
+ }
328
+ if (video.input) {
329
+ if (video.input.location.kind !== 'url' || !video.input.location.url.startsWith('gs://')) {
330
+ throw new VideoError('Veo extends a video in Cloud Storage: give it as a gs:// URL', 'VIDEO_UNSUPPORTED');
331
+ }
332
+ return { gcsUri: video.input.location.url, mimeType: video.input.mimeType };
333
+ }
334
+ const from = video.from;
335
+ if (isInteraction(from.ref)) {
336
+ throw new VideoError('Veo extends only its own videos; give an Omni video as input', 'VIDEO_UNSUPPORTED');
337
+ }
338
+ const sample = samplesOf(await this.operation(from.ref, context))[from.index ?? 0];
339
+ if (!sample?.uri?.startsWith('gs://')) {
340
+ throw new VideoError(`Job ${from.ref.id} kept no video in Cloud Storage to extend; set storageUri so Veo writes there`, 'VIDEO_UNSUPPORTED');
341
+ }
342
+ return { gcsUri: sample.uri, mimeType: sample.mimeType };
343
+ }
164
344
  image(input) {
345
+ if (input.location.kind === 'url' && this.vertex && input.location.url.startsWith('gs://')) {
346
+ return { gcsUri: input.location.url, mimeType: input.mimeType };
347
+ }
165
348
  if (input.location.kind !== 'bytes') {
166
- throw new VideoError('Veo takes an input image as bytes', 'VIDEO_UNSUPPORTED');
349
+ throw new VideoError('Veo takes an input image as bytes, or on Vertex AI from Cloud Storage', 'VIDEO_UNSUPPORTED');
167
350
  }
168
351
  const data = encodeBase64(input.location.data);
169
352
  return this.vertex
@@ -213,6 +396,52 @@ export class GoogleVideoProvider {
213
396
  }
214
397
  }
215
398
  }
399
+ function mergeByModel(known, given) {
400
+ const merged = {};
401
+ for (const [model, changes] of Object.entries(given ?? {}))
402
+ merged[model] = { ...known[model], ...changes };
403
+ return merged;
404
+ }
405
+ function isInteraction(ref) {
406
+ const google = ref.metadata?.google;
407
+ return google?.api === 'interactions' || isOmni(ref.model);
408
+ }
409
+ function media(type, input) {
410
+ if (input.location.kind === 'bytes') {
411
+ return { type, data: encodeBase64(input.location.data), mime_type: input.mimeType };
412
+ }
413
+ if (input.location.kind === 'url')
414
+ return { type, uri: input.location.url, mime_type: input.mimeType };
415
+ throw new VideoError(`Gemini Omni Flash takes an input ${type} as bytes or by its Files API URI`, 'VIDEO_UNSUPPORTED');
416
+ }
417
+ function videosOf(interaction) {
418
+ return (interaction.steps ?? [])
419
+ .filter((step) => step.type === 'model_output')
420
+ .flatMap((step) => step.content ?? [])
421
+ .filter((item) => item.type === 'video' && (item.data || item.uri))
422
+ .map((item) => ({
423
+ ...(item.data ? { base64: item.data } : { uri: item.uri }),
424
+ mimeType: item.mime_type || 'video/mp4',
425
+ }));
426
+ }
427
+ function errorOf(interaction) {
428
+ return interaction.error ?? interaction.steps?.find((step) => step.error)?.error;
429
+ }
430
+ function summary(interaction) {
431
+ return {
432
+ ...interaction,
433
+ ...(interaction.steps
434
+ ? {
435
+ steps: interaction.steps.map((step) => ({
436
+ ...step,
437
+ ...(step.content
438
+ ? { content: step.content.map(({ data, ...item }) => (data ? { ...item, data: '(omitted)' } : item)) }
439
+ : {}),
440
+ })),
441
+ }
442
+ : {}),
443
+ };
444
+ }
216
445
  function samplesOf(operation) {
217
446
  const gemini = operation.response?.generateVideoResponse?.generatedSamples;
218
447
  if (gemini) {
@@ -3,8 +3,27 @@ import type { AssetDescriptor, AssetInput } from '../types/images.js';
3
3
  import type { OperationLifecycleLike, ProviderCallContext } from '../types/lifecycle.js';
4
4
  import type { DurableOperationHandle, OperationRunnerConfig } from '../types/operations.js';
5
5
  import type { ResponseCost } from '../types/response.js';
6
- /** What a video can be made from beside its prompt. */
7
- export type VideoInputKind = 'image' | 'last-frame' | 'references';
6
+ /**
7
+ * What a video can be made from beside its prompt: a first frame, a last frame, reference images, or
8
+ * a video to continue (`extend`) or to change as the prompt says (`edit`).
9
+ */
10
+ export type VideoInputKind = 'image' | 'last-frame' | 'references' | 'extend' | 'edit';
11
+ /**
12
+ * A video to work from: one to continue from its end, or one to change as the prompt says. Give the
13
+ * video itself as `input`, or name a finished job of the same provider as `from`, which the provider
14
+ * continues from what it already holds, with nothing uploaded.
15
+ */
16
+ export interface VideoSource {
17
+ /** `extend` continues the video; `edit` changes it as the prompt says. */
18
+ mode: 'extend' | 'edit';
19
+ /** The video: bytes, a URL the provider can read, or an asset in the manager's store. */
20
+ input?: AssetInput;
21
+ /** Or a video this provider made: its job, and which of the job's videos, from 0. */
22
+ from?: {
23
+ ref: VideoJobRef;
24
+ index?: number;
25
+ };
26
+ }
8
27
  /** Whether a model makes sound: always, when asked, or never. */
9
28
  export type VideoAudioSupport = 'always' | 'optional' | 'none';
10
29
  /** A video to make. */
@@ -21,7 +40,9 @@ export interface VideoGenerateRequest {
21
40
  lastFrame?: AssetInput;
22
41
  /** Images whose subject, character, or style the video keeps. */
23
42
  references?: readonly AssetInput[];
24
- /** Length in seconds. Defaults to the model's. */
43
+ /** A video to extend or edit. */
44
+ video?: VideoSource;
45
+ /** Length in seconds; for `extend`, how much is added. Defaults to the model's. */
25
46
  durationSeconds?: number;
26
47
  /** Frame shape, such as `16:9` or `9:16`. */
27
48
  aspectRatio?: string;
@@ -70,7 +91,9 @@ export interface VideoProviderRequest {
70
91
  lastFrame?: AssetInput;
71
92
  /** Reference images. */
72
93
  references?: readonly AssetInput[];
73
- /** Length in seconds. */
94
+ /** A video to extend or edit, its input resolved to bytes or a URL. */
95
+ video?: VideoSource;
96
+ /** Length in seconds; for `extend`, how much is added. */
74
97
  durationSeconds?: number;
75
98
  /** Frame shape. */
76
99
  aspectRatio?: string;
@@ -170,6 +193,12 @@ export interface VideoModelCapabilities {
170
193
  supportsSeed?: boolean;
171
194
  /** Whether it takes a negative prompt. */
172
195
  supportsNegativePrompt?: boolean;
196
+ /** The lengths `extend` adds, in seconds. Any when absent. */
197
+ extendDurations?: readonly number[];
198
+ /** What every request to it must give, such as `edit` for a model that only changes videos. */
199
+ requires?: readonly VideoInputKind[];
200
+ /** Whether `video.from` may name a job of its own, which it continues from what it holds. */
201
+ continuesJobs?: boolean;
173
202
  }
174
203
  /** What a video provider supports: its models, and what each makes. */
175
204
  export interface VideoProviderCapabilities extends VideoModelCapabilities {
@@ -183,6 +212,8 @@ export interface VideoProviderCapabilities extends VideoModelCapabilities {
183
212
  inputLocationKinds: readonly ('bytes' | 'url')[];
184
213
  /** The image types it takes. Any image when absent. */
185
214
  inputMimeTypes?: readonly string[];
215
+ /** The video types it takes to extend or edit. Any video when absent. */
216
+ inputVideoMimeTypes?: readonly string[];
186
217
  /** Whether a running job can be cancelled. */
187
218
  supportsCancel?: boolean;
188
219
  /** Whether a finished job's video can be deleted from the provider. */
@@ -267,7 +298,9 @@ export interface VideoPrice {
267
298
  }
268
299
  /**
269
300
  * List prices on the Gemini API's paid tier, in dollars per second with sound, as published on
270
- * 2026-10-14. An account priced otherwise passes its own as `prices`.
301
+ * 2026-10-20. Gemini Omni Flash is billed by output token, 5,792 a second of 720p video at $17.50 a
302
+ * million, which is the rate here; Google publishes no rate for its other sizes, which stay unpriced.
303
+ * The Veo 3.1 previews shut down on 2026-10-22. An account priced otherwise passes its own as `prices`.
271
304
  */
272
305
  export declare const VIDEO_PRICES: Readonly<Record<string, VideoPrice>>;
273
306
  /**
@@ -335,7 +368,7 @@ export interface VideoCollectOptions {
335
368
  *
336
369
  * ```ts
337
370
  * const video = new VideoManager(
338
- * { providers: { veo: new GoogleVideoProvider({ apiKey }) }, store: new S3AssetStore({ ... }) },
371
+ * { providers: { google: new GoogleVideoProvider({ apiKey }) }, store: new S3AssetStore({ ... }) },
339
372
  * { operations: { store: new RedisOperationStore({ client }) }, lifecycle: ai.lifecycle },
340
373
  * );
341
374
  * const handle = await video.generate({ prompt: 'a lighthouse at dusk', durationSeconds: 8 });
@@ -388,6 +421,8 @@ export declare class VideoManager {
388
421
  private deliveryFor;
389
422
  /** Checks a request against the provider's model and resolves stored inputs to bytes. */
390
423
  private prepare;
424
+ /** Checks a video to work from: a mode the model takes, and one source, an input or a job of this provider's. */
425
+ private checkVideo;
391
426
  /** An input as the provider takes it: a stored asset read from the manager's store into bytes. */
392
427
  private resolveInput;
393
428
  private resolveProvider;
@@ -412,6 +447,14 @@ export interface VideoConformanceOptions {
412
447
  request?: Partial<VideoProviderRequest>;
413
448
  /** An image, to check image-to-video on a provider that takes one. */
414
449
  image?: AssetInput;
450
+ /**
451
+ * A video of the provider's limits, to check extending or editing it on a model that takes that. A
452
+ * model that continues its own jobs is also asked to extend the first job it made.
453
+ */
454
+ video?: {
455
+ mode: 'extend' | 'edit';
456
+ input: AssetInput;
457
+ };
415
458
  /** Submits and cancels a second job, on a provider that can cancel. Off by default: it costs a job. */
416
459
  checkCancel?: boolean;
417
460
  /** How often to poll. Defaults to 5 seconds. */