nexus-ai-pro 2.16.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +22 -0
- package/CHANGELOG.md +54 -1
- package/README.md +6 -4
- package/dist/agent/capabilities.d.ts +3 -0
- package/dist/mcp/client.d.ts +62 -3
- package/dist/mcp/client.js +154 -16
- package/dist/mcp/index.d.ts +2 -2
- package/dist/mcp/index.js +2 -2
- package/dist/mcp/server.d.ts +12 -0
- package/dist/mcp/server.js +24 -5
- package/dist/protocols/acp-editor.d.ts +16 -2
- package/dist/protocols/acp-editor.js +21 -9
- package/dist/protocols/acp.d.ts +8 -0
- package/dist/protocols/acp.js +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/video/google.d.ts +42 -11
- package/dist/video/google.js +249 -20
- package/dist/video/index.d.ts +49 -6
- package/dist/video/index.js +91 -15
- package/dist/video/mock.js +10 -1
- package/dist/video/runway.d.ts +8 -1
- package/dist/video/runway.js +68 -2
- package/dist-cjs/agent/capabilities.d.ts +3 -0
- package/dist-cjs/mcp/client.d.ts +62 -3
- package/dist-cjs/mcp/client.js +156 -16
- package/dist-cjs/mcp/index.d.ts +2 -2
- package/dist-cjs/mcp/index.js +4 -1
- package/dist-cjs/mcp/server.d.ts +12 -0
- package/dist-cjs/mcp/server.js +25 -5
- package/dist-cjs/protocols/acp-editor.d.ts +16 -2
- package/dist-cjs/protocols/acp-editor.js +20 -8
- package/dist-cjs/protocols/acp.d.ts +8 -0
- package/dist-cjs/protocols/acp.js +1 -0
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/dist-cjs/video/google.d.ts +42 -11
- package/dist-cjs/video/google.js +248 -19
- package/dist-cjs/video/index.d.ts +49 -6
- package/dist-cjs/video/index.js +91 -15
- package/dist-cjs/video/mock.js +10 -1
- package/dist-cjs/video/runway.d.ts +8 -1
- package/dist-cjs/video/runway.js +68 -2
- package/package.json +1 -1
package/dist/video/google.d.ts
CHANGED
|
@@ -11,15 +11,25 @@ export interface GoogleVideoProviderConfig {
|
|
|
11
11
|
accessToken?: string | (() => string | Promise<string>);
|
|
12
12
|
/** API base URL. Defaults to the Gemini API, `https://generativelanguage.googleapis.com/v1beta`. */
|
|
13
13
|
baseUrl?: string;
|
|
14
|
-
/**
|
|
14
|
+
/**
|
|
15
|
+
* Vertex AI only: where Veo writes each video, as `gs://bucket/prefix`. Without it Vertex returns the
|
|
16
|
+
* bytes inline, and a job cannot be extended from what it made.
|
|
17
|
+
*/
|
|
15
18
|
storageUri?: string;
|
|
16
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* The model a request naming none uses. Defaults to `gemini-omni-1.1-flash` on the Gemini API and
|
|
21
|
+
* `veo-3.1-generate-001` on Vertex AI.
|
|
22
|
+
*/
|
|
17
23
|
defaultModel?: string;
|
|
18
|
-
/**
|
|
24
|
+
/**
|
|
25
|
+
* The models served. Defaults to Gemini Omni Flash on the Gemini API, where the Veo 3.1 previews shut
|
|
26
|
+
* down on 2026-10-22, and to Veo 3.1 on Vertex AI. A model named `gemini-omni…` is reached through the
|
|
27
|
+
* Interactions API, any other through Veo's protocol.
|
|
28
|
+
*/
|
|
19
29
|
models?: readonly string[];
|
|
20
|
-
/** Where a model differs from
|
|
30
|
+
/** Where a model differs from what the adapter knows of it. */
|
|
21
31
|
byModel?: Record<string, Partial<VideoModelCapabilities>>;
|
|
22
|
-
/** `dont_allow`, `allow_adult`, or `allow_all`, sent as `personGeneration`. Some regions allow only `allow_adult`. */
|
|
32
|
+
/** `dont_allow`, `allow_adult`, or `allow_all`, sent to Veo as `personGeneration`. Some regions allow only `allow_adult`. */
|
|
23
33
|
personGeneration?: string;
|
|
24
34
|
/** Headers added to every request. */
|
|
25
35
|
defaultHeaders?: Record<string, string>;
|
|
@@ -27,9 +37,15 @@ export interface GoogleVideoProviderConfig {
|
|
|
27
37
|
fetch?: typeof globalThis.fetch;
|
|
28
38
|
}
|
|
29
39
|
/**
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
40
|
+
* Google's video models behind the video contract.
|
|
41
|
+
*
|
|
42
|
+
* Gemini Omni Flash runs as a background interaction: its id is the ref, any worker reads it, and its
|
|
43
|
+
* video comes back with it, inline or by a Files API URI. It extends or edits a video given to it, or
|
|
44
|
+
* continues an interaction of its own by its id, with nothing uploaded.
|
|
45
|
+
*
|
|
46
|
+
* Veo runs as a long-running operation: its name is the ref, any worker polls it, and the video is read
|
|
47
|
+
* from the URI the Gemini API returns, from Cloud Storage, or from the bytes Vertex returns inline. On
|
|
48
|
+
* Vertex it extends a video in Cloud Storage by seven seconds, or a job of its own that wrote there.
|
|
33
49
|
*/
|
|
34
50
|
export declare class GoogleVideoProvider implements VideoProvider {
|
|
35
51
|
private readonly config;
|
|
@@ -38,14 +54,29 @@ export declare class GoogleVideoProvider implements VideoProvider {
|
|
|
38
54
|
private readonly baseUrl;
|
|
39
55
|
private readonly fetchImplementation;
|
|
40
56
|
private readonly vertex;
|
|
57
|
+
/** Interactions read as finished, so the download right after reads no second copy of the video. */
|
|
58
|
+
private readonly finished;
|
|
41
59
|
constructor(config: GoogleVideoProviderConfig);
|
|
42
|
-
/**
|
|
60
|
+
/** What one model takes: Omni's, Veo 3.1's, or Veo's on Vertex, which extends videos too. */
|
|
61
|
+
private capabilitiesFor;
|
|
62
|
+
/** Starts an interaction for Omni, or a long-running prediction for Veo. */
|
|
43
63
|
submit(request: VideoProviderRequest, context: VideoCallContext): Promise<VideoJobRef>;
|
|
44
|
-
/** Reads the operation. */
|
|
64
|
+
/** Reads the interaction or the operation. */
|
|
45
65
|
poll(ref: VideoJobRef, context: VideoCallContext): Promise<VideoJobState>;
|
|
46
|
-
/** Streams one video: from its URI, from Cloud Storage, or from the bytes
|
|
66
|
+
/** Streams one video: from its URI, from Cloud Storage, or from the bytes returned with the job. */
|
|
47
67
|
download(ref: VideoJobRef, index: number, context: VideoCallContext): Promise<VideoContent>;
|
|
68
|
+
/** Cancels a running interaction. A Veo operation runs to its end. */
|
|
69
|
+
cancel(ref: VideoJobRef, context: VideoCallContext): Promise<void>;
|
|
70
|
+
/** Deletes a finished interaction, and its video, from Google. */
|
|
71
|
+
remove(ref: VideoJobRef, context: VideoCallContext): Promise<void>;
|
|
72
|
+
private startInteraction;
|
|
73
|
+
private pollInteraction;
|
|
74
|
+
private interaction;
|
|
75
|
+
/** Keeps the last few finished interactions, whose video the download reads. */
|
|
76
|
+
private remember;
|
|
48
77
|
private operation;
|
|
78
|
+
/** The video Veo extends: one in Cloud Storage, or one a job of its own wrote there. */
|
|
79
|
+
private veoVideo;
|
|
49
80
|
private image;
|
|
50
81
|
private call;
|
|
51
82
|
private send;
|
package/dist/video/google.js
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { decodeBase64, encodeBase64 } from '../utils/base64.js';
|
|
2
2
|
import { VideoError } from './errors.js';
|
|
3
3
|
const GEMINI_API_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta';
|
|
4
4
|
const STORAGE_BASE_URL = 'https://storage.googleapis.com/storage/v1';
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
'veo-3.1-generate-preview',
|
|
8
|
-
'veo-3.1-fast-generate-preview',
|
|
9
|
-
'veo-3.1-lite-generate-preview',
|
|
10
|
-
];
|
|
5
|
+
const OMNI_MODEL = 'gemini-omni-1.1-flash';
|
|
6
|
+
const VERTEX_MODELS = ['veo-3.1-generate-001', 'veo-3.1-fast-generate-001', 'veo-3.1-lite-generate-001'];
|
|
11
7
|
const VEO_31 = {
|
|
12
8
|
inputs: ['image', 'last-frame', 'references'],
|
|
13
9
|
durations: [4, 6, 8],
|
|
@@ -20,12 +16,28 @@ const VEO_31 = {
|
|
|
20
16
|
supportsSeed: true,
|
|
21
17
|
supportsNegativePrompt: true,
|
|
22
18
|
};
|
|
19
|
+
const OMNI_LENGTHS = [3, 4, 5, 6, 7, 8, 9, 10];
|
|
20
|
+
const OMNI = {
|
|
21
|
+
inputs: ['image', 'references', 'extend', 'edit'],
|
|
22
|
+
durations: OMNI_LENGTHS,
|
|
23
|
+
defaultDurationSeconds: 8,
|
|
24
|
+
extendDurations: OMNI_LENGTHS,
|
|
25
|
+
aspectRatios: ['16:9', '9:16'],
|
|
26
|
+
resolutions: ['360p', '720p', '1080p', '4k'],
|
|
27
|
+
audio: 'always',
|
|
28
|
+
maxCount: 1,
|
|
29
|
+
supportsSeed: false,
|
|
30
|
+
supportsNegativePrompt: false,
|
|
31
|
+
continuesJobs: true,
|
|
32
|
+
};
|
|
33
|
+
const isOmni = (model) => model.startsWith('gemini-omni');
|
|
23
34
|
export class GoogleVideoProvider {
|
|
24
35
|
config;
|
|
25
36
|
info;
|
|
26
37
|
baseUrl;
|
|
27
38
|
fetchImplementation;
|
|
28
39
|
vertex;
|
|
40
|
+
finished = new Map();
|
|
29
41
|
constructor(config) {
|
|
30
42
|
this.config = config;
|
|
31
43
|
if (!config.apiKey?.trim() && !config.accessToken) {
|
|
@@ -38,26 +50,50 @@ export class GoogleVideoProvider {
|
|
|
38
50
|
this.fetchImplementation = fetchImplementation;
|
|
39
51
|
this.vertex = config.accessToken !== undefined;
|
|
40
52
|
this.baseUrl = (config.baseUrl || GEMINI_API_BASE_URL).replace(/\/+$/, '');
|
|
41
|
-
const models = config.models ??
|
|
53
|
+
const models = config.models ?? (this.vertex ? VERTEX_MODELS : [OMNI_MODEL]);
|
|
42
54
|
const byModel = {};
|
|
43
|
-
|
|
44
|
-
byModel[
|
|
45
|
-
|
|
55
|
+
for (const model of models)
|
|
56
|
+
byModel[model] = this.capabilitiesFor(model);
|
|
57
|
+
const defaultModel = config.defaultModel ?? models[0];
|
|
58
|
+
const omniOnly = models.every(isOmni);
|
|
46
59
|
this.info = {
|
|
47
60
|
name: 'google',
|
|
48
61
|
capabilities: {
|
|
49
|
-
|
|
50
|
-
|
|
62
|
+
...(defaultModel ? byModel[defaultModel] : VEO_31),
|
|
63
|
+
inputs: (defaultModel ? byModel[defaultModel]?.inputs : undefined) ?? VEO_31.inputs,
|
|
51
64
|
models,
|
|
52
|
-
defaultModel
|
|
53
|
-
byModel: { ...byModel, ...config.byModel },
|
|
54
|
-
inputLocationKinds: ['bytes'],
|
|
65
|
+
...(defaultModel ? { defaultModel } : {}),
|
|
66
|
+
byModel: { ...byModel, ...mergeByModel(byModel, config.byModel) },
|
|
67
|
+
inputLocationKinds: this.vertex || models.some(isOmni) ? ['bytes', 'url'] : ['bytes'],
|
|
55
68
|
inputMimeTypes: ['image/png', 'image/jpeg', 'image/webp'],
|
|
56
|
-
|
|
69
|
+
inputVideoMimeTypes: ['video/mp4'],
|
|
70
|
+
supportsCancel: omniOnly,
|
|
71
|
+
supportsRemove: omniOnly,
|
|
72
|
+
retentionHours: this.vertex ? undefined : omniOnly ? 24 : 48,
|
|
57
73
|
},
|
|
58
74
|
};
|
|
59
75
|
}
|
|
76
|
+
capabilitiesFor(model) {
|
|
77
|
+
if (isOmni(model))
|
|
78
|
+
return OMNI;
|
|
79
|
+
const veo = { ...VEO_31 };
|
|
80
|
+
if (model.includes('lite'))
|
|
81
|
+
veo.resolutions = ['720p', '1080p'];
|
|
82
|
+
if (this.vertex) {
|
|
83
|
+
if (model.includes('fast'))
|
|
84
|
+
veo.resolutions = ['720p', '1080p'];
|
|
85
|
+
if (model.includes('lite'))
|
|
86
|
+
veo.inputs = ['image', 'last-frame'];
|
|
87
|
+
veo.inputs = [...veo.inputs, 'extend'];
|
|
88
|
+
veo.extendDurations = [7];
|
|
89
|
+
veo.maxCount = 4;
|
|
90
|
+
veo.continuesJobs = this.config.storageUri !== undefined;
|
|
91
|
+
}
|
|
92
|
+
return veo;
|
|
93
|
+
}
|
|
60
94
|
async submit(request, context) {
|
|
95
|
+
if (isOmni(request.model))
|
|
96
|
+
return this.startInteraction(request, context);
|
|
61
97
|
const instance = { prompt: request.prompt };
|
|
62
98
|
if (request.image)
|
|
63
99
|
instance.image = this.image(request.image);
|
|
@@ -69,6 +105,8 @@ export class GoogleVideoProvider {
|
|
|
69
105
|
referenceType: typeof reference.metadata?.referenceType === 'string' ? reference.metadata.referenceType : 'asset',
|
|
70
106
|
}));
|
|
71
107
|
}
|
|
108
|
+
if (request.video)
|
|
109
|
+
instance.video = await this.veoVideo(request, context);
|
|
72
110
|
const parameters = {};
|
|
73
111
|
if (request.aspectRatio)
|
|
74
112
|
parameters.aspectRatio = request.aspectRatio;
|
|
@@ -97,6 +135,8 @@ export class GoogleVideoProvider {
|
|
|
97
135
|
return { provider: this.info.name, id: operation.name, model: request.model, createdAt: new Date().toISOString() };
|
|
98
136
|
}
|
|
99
137
|
async poll(ref, context) {
|
|
138
|
+
if (isInteraction(ref))
|
|
139
|
+
return this.pollInteraction(ref, context);
|
|
100
140
|
const operation = await this.operation(ref, context);
|
|
101
141
|
if (!operation.done)
|
|
102
142
|
return { status: 'running', raw: operation };
|
|
@@ -129,7 +169,9 @@ export class GoogleVideoProvider {
|
|
|
129
169
|
return { status: 'completed', outputs: samples.length, ...(filtered ? { filtered } : {}), raw: operation };
|
|
130
170
|
}
|
|
131
171
|
async download(ref, index, context) {
|
|
132
|
-
const sample =
|
|
172
|
+
const sample = isInteraction(ref)
|
|
173
|
+
? videosOf(this.finished.get(ref.id) ?? (await this.interaction(ref.id, context)))[index]
|
|
174
|
+
: samplesOf(await this.operation(ref, context))[index];
|
|
133
175
|
if (!sample)
|
|
134
176
|
throw new VideoError(`Job ${ref.id} has no video ${index}`, 'VIDEO_INVALID');
|
|
135
177
|
if (sample.base64 !== undefined) {
|
|
@@ -143,7 +185,7 @@ export class GoogleVideoProvider {
|
|
|
143
185
|
return { stream, mimeType: sample.mimeType, byteLength: data.byteLength };
|
|
144
186
|
}
|
|
145
187
|
const uri = sample.uri;
|
|
146
|
-
const url = uri.startsWith('gs://') ? storageUrl(uri) : uri
|
|
188
|
+
const url = uri.startsWith('gs://') ? storageUrl(uri) : /^https?:\/\//.test(uri) ? uri : `${this.baseUrl}/${uri}`;
|
|
147
189
|
const response = await this.send(url, { method: 'GET', redirect: 'follow' }, context);
|
|
148
190
|
if (!response.ok || !response.body) {
|
|
149
191
|
throw await failure(response, `Google video download for job ${ref.id} failed`);
|
|
@@ -155,15 +197,156 @@ export class GoogleVideoProvider {
|
|
|
155
197
|
...(Number.isSafeInteger(length) && length > 0 ? { byteLength: length } : {}),
|
|
156
198
|
};
|
|
157
199
|
}
|
|
200
|
+
async cancel(ref, context) {
|
|
201
|
+
if (!isInteraction(ref))
|
|
202
|
+
throw new VideoError('A Veo operation cannot be cancelled here', 'VIDEO_UNSUPPORTED');
|
|
203
|
+
await this.call(`${this.baseUrl}/interactions/${encodeURIComponent(ref.id)}/cancel`, { method: 'POST' }, context);
|
|
204
|
+
}
|
|
205
|
+
async remove(ref, context) {
|
|
206
|
+
if (!isInteraction(ref))
|
|
207
|
+
throw new VideoError('A Veo video expires on its own', 'VIDEO_UNSUPPORTED');
|
|
208
|
+
this.finished.delete(ref.id);
|
|
209
|
+
await this.call(`${this.baseUrl}/interactions/${encodeURIComponent(ref.id)}`, { method: 'DELETE' }, context);
|
|
210
|
+
}
|
|
211
|
+
async startInteraction(request, context) {
|
|
212
|
+
if (this.vertex) {
|
|
213
|
+
throw new VideoError('Gemini Omni Flash is reached through the Gemini API, not Vertex AI', 'VIDEO_UNSUPPORTED');
|
|
214
|
+
}
|
|
215
|
+
const parts = [];
|
|
216
|
+
if (request.video?.input)
|
|
217
|
+
parts.push(media('video', request.video.input));
|
|
218
|
+
if (request.image)
|
|
219
|
+
parts.push(media('image', request.image));
|
|
220
|
+
for (const reference of request.references ?? [])
|
|
221
|
+
parts.push(media('image', reference));
|
|
222
|
+
if (request.prompt.trim())
|
|
223
|
+
parts.push({ type: 'text', text: request.prompt });
|
|
224
|
+
const format = { type: 'video' };
|
|
225
|
+
if (request.aspectRatio)
|
|
226
|
+
format.aspect_ratio = request.aspectRatio;
|
|
227
|
+
if (request.resolution)
|
|
228
|
+
format.resolution = request.resolution;
|
|
229
|
+
if (request.durationSeconds !== undefined)
|
|
230
|
+
format.duration = `${request.durationSeconds}s`;
|
|
231
|
+
const body = {
|
|
232
|
+
model: request.model,
|
|
233
|
+
input: parts.length === 1 && parts[0]?.type === 'text' ? request.prompt : parts,
|
|
234
|
+
response_format: format,
|
|
235
|
+
background: true,
|
|
236
|
+
store: true,
|
|
237
|
+
};
|
|
238
|
+
const from = request.video?.from;
|
|
239
|
+
if (from) {
|
|
240
|
+
if (!isInteraction(from.ref)) {
|
|
241
|
+
throw new VideoError('Gemini Omni Flash continues only its own interactions; give a Veo video as input', 'VIDEO_UNSUPPORTED');
|
|
242
|
+
}
|
|
243
|
+
if ((from.index ?? 0) !== 0)
|
|
244
|
+
throw new VideoError('An interaction makes one video, at index 0', 'VIDEO_INVALID');
|
|
245
|
+
body.previous_interaction_id = from.ref.id;
|
|
246
|
+
}
|
|
247
|
+
const task = request.video
|
|
248
|
+
? request.video.mode
|
|
249
|
+
: request.references?.length
|
|
250
|
+
? 'reference_to_video'
|
|
251
|
+
: request.image
|
|
252
|
+
? 'image_to_video'
|
|
253
|
+
: undefined;
|
|
254
|
+
if (task)
|
|
255
|
+
body.generation_config = { video_config: { task } };
|
|
256
|
+
Object.assign(body, request.providerOptions);
|
|
257
|
+
const interaction = await this.call(`${this.baseUrl}/interactions`, { method: 'POST', body: JSON.stringify(body) }, context);
|
|
258
|
+
if (!interaction.id)
|
|
259
|
+
throw new VideoError('Google returned no interaction id', 'VIDEO_PROVIDER');
|
|
260
|
+
return {
|
|
261
|
+
provider: this.info.name,
|
|
262
|
+
id: interaction.id,
|
|
263
|
+
model: request.model,
|
|
264
|
+
createdAt: new Date().toISOString(),
|
|
265
|
+
metadata: { google: { api: 'interactions' } },
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
async pollInteraction(ref, context) {
|
|
269
|
+
const interaction = await this.interaction(ref.id, context);
|
|
270
|
+
switch (interaction.status) {
|
|
271
|
+
case 'queued':
|
|
272
|
+
return { status: 'queued', raw: summary(interaction) };
|
|
273
|
+
case 'in_progress':
|
|
274
|
+
case 'requires_action':
|
|
275
|
+
return { status: 'running', raw: summary(interaction) };
|
|
276
|
+
case 'cancelled':
|
|
277
|
+
return { status: 'cancelled', raw: summary(interaction) };
|
|
278
|
+
case 'completed': {
|
|
279
|
+
const videos = videosOf(interaction);
|
|
280
|
+
if (videos.length === 0) {
|
|
281
|
+
return {
|
|
282
|
+
status: 'failed',
|
|
283
|
+
error: { message: errorOf(interaction)?.message ?? 'the interaction made no video', retryable: false },
|
|
284
|
+
raw: summary(interaction),
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
this.remember(ref.id, interaction);
|
|
288
|
+
return { status: 'completed', outputs: videos.length, raw: summary(interaction) };
|
|
289
|
+
}
|
|
290
|
+
default: {
|
|
291
|
+
const error = errorOf(interaction);
|
|
292
|
+
const code = error?.status ?? (error?.code === undefined ? interaction.status : String(error.code));
|
|
293
|
+
return {
|
|
294
|
+
status: 'failed',
|
|
295
|
+
error: {
|
|
296
|
+
message: error?.message ?? `the interaction ended ${interaction.status ?? 'without a status'}`,
|
|
297
|
+
...(code ? { code } : {}),
|
|
298
|
+
retryable: [13, 14, 4].includes(error?.code ?? -1),
|
|
299
|
+
...(/SAFETY|PROHIBITED|BLOCK/i.test(`${error?.status ?? ''} ${error?.message ?? ''}`)
|
|
300
|
+
? { safety: true }
|
|
301
|
+
: {}),
|
|
302
|
+
},
|
|
303
|
+
raw: summary(interaction),
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
interaction(id, context) {
|
|
309
|
+
return this.call(`${this.baseUrl}/interactions/${encodeURIComponent(id)}`, { method: 'GET' }, context);
|
|
310
|
+
}
|
|
311
|
+
remember(id, interaction) {
|
|
312
|
+
this.finished.delete(id);
|
|
313
|
+
this.finished.set(id, interaction);
|
|
314
|
+
while (this.finished.size > 4)
|
|
315
|
+
this.finished.delete(this.finished.keys().next().value);
|
|
316
|
+
}
|
|
158
317
|
operation(ref, context) {
|
|
159
318
|
if (this.vertex) {
|
|
160
319
|
return this.call(`${this.baseUrl}/models/${encodeURIComponent(ref.model)}:fetchPredictOperation`, { method: 'POST', body: JSON.stringify({ operationName: ref.id }) }, context);
|
|
161
320
|
}
|
|
162
321
|
return this.call(`${this.baseUrl}/${ref.id}`, { method: 'GET' }, context);
|
|
163
322
|
}
|
|
323
|
+
async veoVideo(request, context) {
|
|
324
|
+
const video = request.video;
|
|
325
|
+
if (!this.vertex || video?.mode !== 'extend') {
|
|
326
|
+
throw new VideoError('Veo extends videos on Vertex AI, and edits none', 'VIDEO_UNSUPPORTED');
|
|
327
|
+
}
|
|
328
|
+
if (video.input) {
|
|
329
|
+
if (video.input.location.kind !== 'url' || !video.input.location.url.startsWith('gs://')) {
|
|
330
|
+
throw new VideoError('Veo extends a video in Cloud Storage: give it as a gs:// URL', 'VIDEO_UNSUPPORTED');
|
|
331
|
+
}
|
|
332
|
+
return { gcsUri: video.input.location.url, mimeType: video.input.mimeType };
|
|
333
|
+
}
|
|
334
|
+
const from = video.from;
|
|
335
|
+
if (isInteraction(from.ref)) {
|
|
336
|
+
throw new VideoError('Veo extends only its own videos; give an Omni video as input', 'VIDEO_UNSUPPORTED');
|
|
337
|
+
}
|
|
338
|
+
const sample = samplesOf(await this.operation(from.ref, context))[from.index ?? 0];
|
|
339
|
+
if (!sample?.uri?.startsWith('gs://')) {
|
|
340
|
+
throw new VideoError(`Job ${from.ref.id} kept no video in Cloud Storage to extend; set storageUri so Veo writes there`, 'VIDEO_UNSUPPORTED');
|
|
341
|
+
}
|
|
342
|
+
return { gcsUri: sample.uri, mimeType: sample.mimeType };
|
|
343
|
+
}
|
|
164
344
|
image(input) {
|
|
345
|
+
if (input.location.kind === 'url' && this.vertex && input.location.url.startsWith('gs://')) {
|
|
346
|
+
return { gcsUri: input.location.url, mimeType: input.mimeType };
|
|
347
|
+
}
|
|
165
348
|
if (input.location.kind !== 'bytes') {
|
|
166
|
-
throw new VideoError('Veo takes an input image as bytes', 'VIDEO_UNSUPPORTED');
|
|
349
|
+
throw new VideoError('Veo takes an input image as bytes, or on Vertex AI from Cloud Storage', 'VIDEO_UNSUPPORTED');
|
|
167
350
|
}
|
|
168
351
|
const data = encodeBase64(input.location.data);
|
|
169
352
|
return this.vertex
|
|
@@ -213,6 +396,52 @@ export class GoogleVideoProvider {
|
|
|
213
396
|
}
|
|
214
397
|
}
|
|
215
398
|
}
|
|
399
|
+
function mergeByModel(known, given) {
|
|
400
|
+
const merged = {};
|
|
401
|
+
for (const [model, changes] of Object.entries(given ?? {}))
|
|
402
|
+
merged[model] = { ...known[model], ...changes };
|
|
403
|
+
return merged;
|
|
404
|
+
}
|
|
405
|
+
function isInteraction(ref) {
|
|
406
|
+
const google = ref.metadata?.google;
|
|
407
|
+
return google?.api === 'interactions' || isOmni(ref.model);
|
|
408
|
+
}
|
|
409
|
+
function media(type, input) {
|
|
410
|
+
if (input.location.kind === 'bytes') {
|
|
411
|
+
return { type, data: encodeBase64(input.location.data), mime_type: input.mimeType };
|
|
412
|
+
}
|
|
413
|
+
if (input.location.kind === 'url')
|
|
414
|
+
return { type, uri: input.location.url, mime_type: input.mimeType };
|
|
415
|
+
throw new VideoError(`Gemini Omni Flash takes an input ${type} as bytes or by its Files API URI`, 'VIDEO_UNSUPPORTED');
|
|
416
|
+
}
|
|
417
|
+
function videosOf(interaction) {
|
|
418
|
+
return (interaction.steps ?? [])
|
|
419
|
+
.filter((step) => step.type === 'model_output')
|
|
420
|
+
.flatMap((step) => step.content ?? [])
|
|
421
|
+
.filter((item) => item.type === 'video' && (item.data || item.uri))
|
|
422
|
+
.map((item) => ({
|
|
423
|
+
...(item.data ? { base64: item.data } : { uri: item.uri }),
|
|
424
|
+
mimeType: item.mime_type || 'video/mp4',
|
|
425
|
+
}));
|
|
426
|
+
}
|
|
427
|
+
function errorOf(interaction) {
|
|
428
|
+
return interaction.error ?? interaction.steps?.find((step) => step.error)?.error;
|
|
429
|
+
}
|
|
430
|
+
function summary(interaction) {
|
|
431
|
+
return {
|
|
432
|
+
...interaction,
|
|
433
|
+
...(interaction.steps
|
|
434
|
+
? {
|
|
435
|
+
steps: interaction.steps.map((step) => ({
|
|
436
|
+
...step,
|
|
437
|
+
...(step.content
|
|
438
|
+
? { content: step.content.map(({ data, ...item }) => (data ? { ...item, data: '(omitted)' } : item)) }
|
|
439
|
+
: {}),
|
|
440
|
+
})),
|
|
441
|
+
}
|
|
442
|
+
: {}),
|
|
443
|
+
};
|
|
444
|
+
}
|
|
216
445
|
function samplesOf(operation) {
|
|
217
446
|
const gemini = operation.response?.generateVideoResponse?.generatedSamples;
|
|
218
447
|
if (gemini) {
|
package/dist/video/index.d.ts
CHANGED
|
@@ -3,8 +3,27 @@ import type { AssetDescriptor, AssetInput } from '../types/images.js';
|
|
|
3
3
|
import type { OperationLifecycleLike, ProviderCallContext } from '../types/lifecycle.js';
|
|
4
4
|
import type { DurableOperationHandle, OperationRunnerConfig } from '../types/operations.js';
|
|
5
5
|
import type { ResponseCost } from '../types/response.js';
|
|
6
|
-
/**
|
|
7
|
-
|
|
6
|
+
/**
|
|
7
|
+
* What a video can be made from beside its prompt: a first frame, a last frame, reference images, or
|
|
8
|
+
* a video to continue (`extend`) or to change as the prompt says (`edit`).
|
|
9
|
+
*/
|
|
10
|
+
export type VideoInputKind = 'image' | 'last-frame' | 'references' | 'extend' | 'edit';
|
|
11
|
+
/**
|
|
12
|
+
* A video to work from: one to continue from its end, or one to change as the prompt says. Give the
|
|
13
|
+
* video itself as `input`, or name a finished job of the same provider as `from`, which the provider
|
|
14
|
+
* continues from what it already holds, with nothing uploaded.
|
|
15
|
+
*/
|
|
16
|
+
export interface VideoSource {
|
|
17
|
+
/** `extend` continues the video; `edit` changes it as the prompt says. */
|
|
18
|
+
mode: 'extend' | 'edit';
|
|
19
|
+
/** The video: bytes, a URL the provider can read, or an asset in the manager's store. */
|
|
20
|
+
input?: AssetInput;
|
|
21
|
+
/** Or a video this provider made: its job, and which of the job's videos, from 0. */
|
|
22
|
+
from?: {
|
|
23
|
+
ref: VideoJobRef;
|
|
24
|
+
index?: number;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
8
27
|
/** Whether a model makes sound: always, when asked, or never. */
|
|
9
28
|
export type VideoAudioSupport = 'always' | 'optional' | 'none';
|
|
10
29
|
/** A video to make. */
|
|
@@ -21,7 +40,9 @@ export interface VideoGenerateRequest {
|
|
|
21
40
|
lastFrame?: AssetInput;
|
|
22
41
|
/** Images whose subject, character, or style the video keeps. */
|
|
23
42
|
references?: readonly AssetInput[];
|
|
24
|
-
/**
|
|
43
|
+
/** A video to extend or edit. */
|
|
44
|
+
video?: VideoSource;
|
|
45
|
+
/** Length in seconds; for `extend`, how much is added. Defaults to the model's. */
|
|
25
46
|
durationSeconds?: number;
|
|
26
47
|
/** Frame shape, such as `16:9` or `9:16`. */
|
|
27
48
|
aspectRatio?: string;
|
|
@@ -70,7 +91,9 @@ export interface VideoProviderRequest {
|
|
|
70
91
|
lastFrame?: AssetInput;
|
|
71
92
|
/** Reference images. */
|
|
72
93
|
references?: readonly AssetInput[];
|
|
73
|
-
/**
|
|
94
|
+
/** A video to extend or edit, its input resolved to bytes or a URL. */
|
|
95
|
+
video?: VideoSource;
|
|
96
|
+
/** Length in seconds; for `extend`, how much is added. */
|
|
74
97
|
durationSeconds?: number;
|
|
75
98
|
/** Frame shape. */
|
|
76
99
|
aspectRatio?: string;
|
|
@@ -170,6 +193,12 @@ export interface VideoModelCapabilities {
|
|
|
170
193
|
supportsSeed?: boolean;
|
|
171
194
|
/** Whether it takes a negative prompt. */
|
|
172
195
|
supportsNegativePrompt?: boolean;
|
|
196
|
+
/** The lengths `extend` adds, in seconds. Any when absent. */
|
|
197
|
+
extendDurations?: readonly number[];
|
|
198
|
+
/** What every request to it must give, such as `edit` for a model that only changes videos. */
|
|
199
|
+
requires?: readonly VideoInputKind[];
|
|
200
|
+
/** Whether `video.from` may name a job of its own, which it continues from what it holds. */
|
|
201
|
+
continuesJobs?: boolean;
|
|
173
202
|
}
|
|
174
203
|
/** What a video provider supports: its models, and what each makes. */
|
|
175
204
|
export interface VideoProviderCapabilities extends VideoModelCapabilities {
|
|
@@ -183,6 +212,8 @@ export interface VideoProviderCapabilities extends VideoModelCapabilities {
|
|
|
183
212
|
inputLocationKinds: readonly ('bytes' | 'url')[];
|
|
184
213
|
/** The image types it takes. Any image when absent. */
|
|
185
214
|
inputMimeTypes?: readonly string[];
|
|
215
|
+
/** The video types it takes to extend or edit. Any video when absent. */
|
|
216
|
+
inputVideoMimeTypes?: readonly string[];
|
|
186
217
|
/** Whether a running job can be cancelled. */
|
|
187
218
|
supportsCancel?: boolean;
|
|
188
219
|
/** Whether a finished job's video can be deleted from the provider. */
|
|
@@ -267,7 +298,9 @@ export interface VideoPrice {
|
|
|
267
298
|
}
|
|
268
299
|
/**
|
|
269
300
|
* List prices on the Gemini API's paid tier, in dollars per second with sound, as published on
|
|
270
|
-
* 2026-10-
|
|
301
|
+
* 2026-10-20. Gemini Omni Flash is billed by output token, 5,792 a second of 720p video at $17.50 a
|
|
302
|
+
* million, which is the rate here; Google publishes no rate for its other sizes, which stay unpriced.
|
|
303
|
+
* The Veo 3.1 previews shut down on 2026-10-22. An account priced otherwise passes its own as `prices`.
|
|
271
304
|
*/
|
|
272
305
|
export declare const VIDEO_PRICES: Readonly<Record<string, VideoPrice>>;
|
|
273
306
|
/**
|
|
@@ -335,7 +368,7 @@ export interface VideoCollectOptions {
|
|
|
335
368
|
*
|
|
336
369
|
* ```ts
|
|
337
370
|
* const video = new VideoManager(
|
|
338
|
-
* { providers: {
|
|
371
|
+
* { providers: { google: new GoogleVideoProvider({ apiKey }) }, store: new S3AssetStore({ ... }) },
|
|
339
372
|
* { operations: { store: new RedisOperationStore({ client }) }, lifecycle: ai.lifecycle },
|
|
340
373
|
* );
|
|
341
374
|
* const handle = await video.generate({ prompt: 'a lighthouse at dusk', durationSeconds: 8 });
|
|
@@ -388,6 +421,8 @@ export declare class VideoManager {
|
|
|
388
421
|
private deliveryFor;
|
|
389
422
|
/** Checks a request against the provider's model and resolves stored inputs to bytes. */
|
|
390
423
|
private prepare;
|
|
424
|
+
/** Checks a video to work from: a mode the model takes, and one source, an input or a job of this provider's. */
|
|
425
|
+
private checkVideo;
|
|
391
426
|
/** An input as the provider takes it: a stored asset read from the manager's store into bytes. */
|
|
392
427
|
private resolveInput;
|
|
393
428
|
private resolveProvider;
|
|
@@ -412,6 +447,14 @@ export interface VideoConformanceOptions {
|
|
|
412
447
|
request?: Partial<VideoProviderRequest>;
|
|
413
448
|
/** An image, to check image-to-video on a provider that takes one. */
|
|
414
449
|
image?: AssetInput;
|
|
450
|
+
/**
|
|
451
|
+
* A video of the provider's limits, to check extending or editing it on a model that takes that. A
|
|
452
|
+
* model that continues its own jobs is also asked to extend the first job it made.
|
|
453
|
+
*/
|
|
454
|
+
video?: {
|
|
455
|
+
mode: 'extend' | 'edit';
|
|
456
|
+
input: AssetInput;
|
|
457
|
+
};
|
|
415
458
|
/** Submits and cancels a second job, on a provider that can cancel. Off by default: it costs a job. */
|
|
416
459
|
checkCancel?: boolean;
|
|
417
460
|
/** How often to poll. Defaults to 5 seconds. */
|