@kolbo/mcp 1.76.2 → 1.76.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -110,6 +110,7 @@ These elevate rich cinematic / reference-anchored sequences. For a short, tight,
|
|
|
110
110
|
## Dialogue & expression
|
|
111
111
|
|
|
112
112
|
- Dialogue goes in quotes and may be in ANY language (Hebrew included). For silent tension, deliver it as expression, not speech: `He does not speak. His expression clearly says: "…"`.
|
|
113
|
+
- **Seedance PERFORMS quoted dialogue natively** — synced voices, lip movement, and room tone come out of the video model itself. Never route scene dialogue through TTS (`generate_speech`) or `generate_lipsync`; write each line in quotes inside its shot beat (`DANIEL says: "…"`) and generate once.
|
|
113
114
|
|
|
114
115
|
## Content tone
|
|
115
116
|
|
|
@@ -9,7 +9,7 @@ Load this file when the user wants a **Seedance 2.5** video (they said "2.5" / "
|
|
|
9
9
|
|
|
10
10
|
**Kolbo MCP routing:** `generate_video` or `generate_elements` (refs / Visual DNA / first-last). Run `list_models({ type: "text_to_video" })` and pick the Seedance 2.5 variant by name.
|
|
11
11
|
|
|
12
|
-
**Audio:** Seedance 2.5 still emits real synced audio. `list_models` may show `sound_generation_type: none` because there is no in-app toggle (`sound_baked_in: true`). Do not tell the user the model is silent.
|
|
12
|
+
**Audio:** Seedance 2.5 still emits real synced audio. `list_models` may show `sound_generation_type: none` because there is no in-app toggle (`sound_baked_in: true`). Do not tell the user the model is silent. Quoted dialogue in the prompt is PERFORMED — synced voices, lip movement, room tone — so scene dialogue never goes through `generate_speech` or `generate_lipsync`; write the lines in quotes inside their shot beats.
|
|
13
13
|
|
|
14
14
|
## What's NEW in 2.5 (verified — never hedge)
|
|
15
15
|
|
package/src/apps/theme.js
CHANGED
|
@@ -144,20 +144,31 @@ body {
|
|
|
144
144
|
.k-skel.portrait { aspect-ratio: 3 / 4; }
|
|
145
145
|
.k-skel::after {
|
|
146
146
|
content: ''; position: absolute; inset: 0;
|
|
147
|
+
/* The shimmer covers the WHOLE cell and outlives the skeleton — .k-skel.done
|
|
148
|
+
only clears its paint, not its box. A pseudo-element hit-tests as its
|
|
149
|
+
originating element, so every click on a finished cell landed on the cell
|
|
150
|
+
instead of the media inside it: image cells still worked (their handler is
|
|
151
|
+
ON the cell), but a <video>'s native controls never saw a single event —
|
|
152
|
+
play, scrub, volume and fullscreen were all dead. It is decoration; it must
|
|
153
|
+
never take a click. */
|
|
154
|
+
pointer-events: none;
|
|
147
155
|
background: linear-gradient(90deg, transparent 0%, rgba(255,255,255,0.06) 40%, rgba(255,255,255,0.10) 50%, rgba(255,255,255,0.06) 60%, transparent 100%);
|
|
148
156
|
background-size: 200% 100%;
|
|
149
157
|
animation: k-sweep 1.6s ease-in-out infinite;
|
|
150
158
|
}
|
|
151
159
|
@keyframes k-sweep { 0% { background-position: 200% 0; } 100% { background-position: -200% 0; } }
|
|
152
160
|
/* Batch grid: per-cell prompt caption + a cell that already finished */
|
|
153
|
-
|
|
161
|
+
/* bottom: 0 puts this exactly where a <video> draws its control bar, so it has
|
|
162
|
+
to be click-through too — the caption is a label, not a target. The full text
|
|
163
|
+
stays reachable: the title tooltip moved onto the cell. */
|
|
164
|
+
.k-skel-cap { position: absolute; left: 0; right: 0; bottom: 0; z-index: 2; pointer-events: none;
|
|
154
165
|
padding: 12px 8px 6px; font-size: 10.5px; color: #fff;
|
|
155
166
|
background: linear-gradient(transparent, rgba(0, 0, 0, 0.65));
|
|
156
167
|
white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
|
157
168
|
.k-skel.done::after { animation: none; background: none; }
|
|
158
169
|
.k-cell-fill { width: 100%; height: 100%; object-fit: contain; display: block; background: #000; }
|
|
159
170
|
.k-gen-badge {
|
|
160
|
-
position: absolute; top: 10px; left: 10px; z-index: 2;
|
|
171
|
+
position: absolute; top: 10px; left: 10px; z-index: 2; pointer-events: none;
|
|
161
172
|
display: inline-flex; align-items: center; gap: 6px;
|
|
162
173
|
padding: 4px 10px; border-radius: 999px;
|
|
163
174
|
background: rgba(0, 0, 0, 0.65); /* no backdrop-filter: nested blur over the
|
|
@@ -586,8 +586,12 @@ function fillBatchCell(sc, i, g) {
|
|
|
586
586
|
cell.classList.add('done');
|
|
587
587
|
var cap = (sc.prompts && sc.prompts[i])
|
|
588
588
|
? '<span class="k-skel-cap" title="' + esc(sc.prompts[i]) + '">' + esc(sc.prompts[i]) + '</span>' : '';
|
|
589
|
+
if (sc.prompts && sc.prompts[i]) cell.title = sc.prompts[i];
|
|
590
|
+
// The controls attribute is not optional here: this cell is a FINISHED result
|
|
591
|
+
// the user is meant to watch, and without it the tile was a muted poster with
|
|
592
|
+
// no way to play, seek or unmute until the whole batch finished and repainted.
|
|
589
593
|
cell.innerHTML = (sc.kind === 'video'
|
|
590
|
-
? '<video class="k-cell-fill" src="' + esc(u) + '"' + (r.thumbnail_url ? ' poster="' + esc(r.thumbnail_url) + '"' : '') + '
|
|
594
|
+
? '<video class="k-cell-fill" src="' + esc(u) + '"' + (r.thumbnail_url ? ' poster="' + esc(r.thumbnail_url) + '"' : '') + ' controls playsinline preload="metadata"></video>'
|
|
591
595
|
: '<img class="k-cell-fill" src="' + esc(u) + '" alt="">') + cap;
|
|
592
596
|
window.kolbo.notifySize();
|
|
593
597
|
}
|
|
@@ -767,7 +771,8 @@ function renderBatchGrid(sc) {
|
|
|
767
771
|
var shape = items[0].type === 'video' ? 'video' : 'square';
|
|
768
772
|
el('stage').innerHTML = '<div class="k-gen-grid n' + Math.min(items.length, 4) + '">' +
|
|
769
773
|
items.map(function (it, i) {
|
|
770
|
-
return '<div class="k-skel done ' + shape + '" data-focus="' + i + '"
|
|
774
|
+
return '<div class="k-skel done ' + shape + '" data-focus="' + i + '"' +
|
|
775
|
+
(it.label ? ' title="' + esc(it.label) + '"' : '') + '>' +
|
|
771
776
|
(it.type === 'video'
|
|
772
777
|
? '<video class="k-cell-fill" src="' + esc(it.url) + '" controls playsinline preload="metadata"></video>'
|
|
773
778
|
: '<img class="k-cell-fill" src="' + esc(it.url) + '" alt="" loading="lazy" style="cursor:zoom-in">') +
|
|
@@ -803,7 +808,8 @@ function renderStatusGrid(sc) {
|
|
|
803
808
|
// file — the tool only knows a url came back, not what kind it is.
|
|
804
809
|
it.kind = refKind(it.url, 'image');
|
|
805
810
|
var shape = it.kind === 'video' ? 'video' : 'square';
|
|
806
|
-
return '<div class="k-skel done ' + shape + '" data-focus="' + i + '"
|
|
811
|
+
return '<div class="k-skel done ' + shape + '" data-focus="' + i + '"' +
|
|
812
|
+
(it.title ? ' title="' + esc(it.title) + '"' : '') + '>' +
|
|
807
813
|
(it.kind === 'video'
|
|
808
814
|
? '<video class="k-cell-fill" src="' + esc(it.url) + '" controls playsinline preload="metadata"></video>'
|
|
809
815
|
: it.kind === 'audio'
|
package/src/tools/generate.js
CHANGED
|
@@ -119,6 +119,22 @@ function mediaKind(url) {
|
|
|
119
119
|
return 'image';
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
const isUrlSource = (source) => typeof source === 'string' && /^https?:\/\//i.test(source);
|
|
123
|
+
|
|
124
|
+
// Which multipart kind a LOCAL file should upload as. mediaKind answers for the
|
|
125
|
+
// card; the upload routes only know these three, and the kind decides the
|
|
126
|
+
// content type a receiving controller dispatches on (kolbo-api's elements
|
|
127
|
+
// controller buckets req.files by `file.mimetype`), so guessing 'image' for
|
|
128
|
+
// everything filed a local .mp4 as a reference image.
|
|
129
|
+
const uploadKind = (source) => {
|
|
130
|
+
const kind = mediaKind(source);
|
|
131
|
+
return kind === 'video' || kind === 'audio' ? kind : 'image';
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// The elements route's own multer ceiling (kolbo-api sdk/index.js). Cap here so
|
|
135
|
+
// an oversized file fails with a readable message instead of after the upload.
|
|
136
|
+
const ELEMENTS_MAX_UPLOAD_BYTES = 200 * 1024 * 1024;
|
|
137
|
+
|
|
122
138
|
// ─── Widget settings block ──────────────────────────────────────────────────
|
|
123
139
|
// What the CALLER actually asked for, for the generation card AND for the model
|
|
124
140
|
// reading the tool result. Undefined/false keys are dropped by JSON.stringify, so
|
|
@@ -1165,11 +1181,11 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1165
1181
|
{
|
|
1166
1182
|
prompt: z.string().describe('Locked Intro prompt (Seedance/Elements): Total line, [GLOBAL LOOK], [CAST] with @ExactDNAName for every visual_dna_ids entry, [LOCATION], then SHOT N. Not SCENE CONTEXT/OPTICS/ACTION packs. Never substitute "the left man" or "Zohar\'s" for @Name.'),
|
|
1167
1183
|
model: z.string().optional().describe('Model identifier. If the user already named a family (Grok / Kling / Veo / Seedance / …), pass THAT family — never default to Seedance because Elements often uses it. Use list_models type="elements" for exact ids and elements_max_* caps. Do NOT omit (omitting = Smart Select).'),
|
|
1168
|
-
reference_images: z.array(z.string()).optional().describe('Array of
|
|
1169
|
-
reference_videos: z.array(z.string()).optional().describe('Array of reference
|
|
1170
|
-
reference_audio_urls: z.array(z.string()).optional().describe('Array of reference audio
|
|
1171
|
-
audio_url: z.string().optional().describe('
|
|
1172
|
-
files: z.array(z.string()).optional().describe('
|
|
1184
|
+
reference_images: z.array(z.string()).optional().describe('Array of image references (product shots, character references, etc.). Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Cap: pass at most `elements_max_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.**'),
|
|
1185
|
+
reference_videos: z.array(z.string()).optional().describe('Array of reference videos for models that accept video inputs. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Cap: pass at most `elements_max_videos` URLs from list_models — if the cap is 0 the model rejects videos.**'),
|
|
1186
|
+
reference_audio_urls: z.array(z.string()).optional().describe('Array of reference audio tracks for models that accept audio inputs. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Cap: pass at most `elements_max_audio` URLs from list_models.** `audio_url` remains supported as the legacy single-track form.'),
|
|
1187
|
+
audio_url: z.string().optional().describe('A single reference audio track — legacy form of reference_audio_urls. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Audio constraints: `elements_max_audio` from list_models gates whether audio is accepted at all; audio duration must fall within `min_audio_duration`-`max_audio_duration`; format must be in `supported_audio_formats` (if specified).**'),
|
|
1188
|
+
files: z.array(z.string()).optional().describe('Untyped catch-all for mixed media — images, videos AND audio, each a URL or an absolute local path. The kind is detected from the file extension and the item is routed to the matching reference list, so a local .mp4 is sent as a video and a local .mp3 as audio. Prefer the typed lists (reference_images / reference_videos / reference_audio_urls) when you already know the kind; they accept local paths too. URLs given here are forwarded as URLs, never re-uploaded. **Caps still apply per kind: `elements_max_images` / `elements_max_videos` / `elements_max_audio` from list_models. Local uploads are capped at 200MB each.**'),
|
|
1173
1189
|
duration: z.number().optional().describe('Output duration in seconds. Must be in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. Default: 5'),
|
|
1174
1190
|
aspect_ratio: z.string().optional().describe('Aspect ratio (e.g., "16:9", "9:16", "1:1"). Must be in `supported_aspect_ratios` from list_models. Default: "16:9"'),
|
|
1175
1191
|
motion: z.string().optional().describe('Motion style / intensity hint (optional)'),
|
|
@@ -1192,40 +1208,105 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1192
1208
|
model = await canonicalModelId(client, model, 'elements'); // lenient id resolution ("z-image" → "z-image/turbo")
|
|
1193
1209
|
if (!prompt) throw new Error('prompt is required');
|
|
1194
1210
|
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1211
|
+
// Elements is the one tool that takes all three modalities, and either a
|
|
1212
|
+
// URL or a local path for any of them. Bucket every input by the kind the
|
|
1213
|
+
// API expects, then split by transport:
|
|
1214
|
+
// URL → forwarded AS a url. Downloading one only to re-upload it
|
|
1215
|
+
// costs the round-trip, risks the 200MB multer ceiling, and
|
|
1216
|
+
// duplicates an asset that already lives on the CDN. `files`
|
|
1217
|
+
// used to do exactly that to every URL handed to it.
|
|
1218
|
+
// local → the only thing that becomes a multipart part, uploaded under
|
|
1219
|
+
// its real kind. `files` forced 'image' on everything, so a
|
|
1220
|
+
// local .mp4/.mp3 arrived with the wrong content type and the
|
|
1221
|
+
// elements controller — which buckets req.files by mimetype —
|
|
1222
|
+
// never saw it as video or audio.
|
|
1223
|
+
const media = { image: [], video: [], audio: [] };
|
|
1224
|
+
const collect = (list, forced) => {
|
|
1225
|
+
for (const src of Array.isArray(list) ? list : (list ? [list] : [])) {
|
|
1226
|
+
if (typeof src !== 'string' || !src.trim()) continue;
|
|
1227
|
+
// The typed lists declare their own kind; `files` is the untyped
|
|
1228
|
+
// catch-all, so it is classified by extension.
|
|
1229
|
+
const bucket = media[forced || uploadKind(src)];
|
|
1230
|
+
if (!bucket.includes(src)) bucket.push(src);
|
|
1231
|
+
}
|
|
1232
|
+
};
|
|
1233
|
+
collect(reference_images, 'image');
|
|
1234
|
+
collect(reference_videos, 'video');
|
|
1235
|
+
collect(reference_audio_urls, 'audio');
|
|
1236
|
+
collect(audio_url, 'audio'); // legacy single-track form; the API merges it into audioUrls anyway
|
|
1237
|
+
collect(files);
|
|
1238
|
+
|
|
1239
|
+
const urlsOf = (kind) => media[kind].filter(isUrlSource);
|
|
1240
|
+
const locals = ['image', 'video', 'audio']
|
|
1241
|
+
.flatMap((kind) => media[kind].filter((src) => !isUrlSource(src)).map((src) => ({ src, kind })));
|
|
1242
|
+
const some = (list) => (list.length ? list : undefined);
|
|
1243
|
+
const body = {
|
|
1244
|
+
prompt, model,
|
|
1245
|
+
reference_images: some(urlsOf('image')),
|
|
1246
|
+
reference_videos: some(urlsOf('video')),
|
|
1247
|
+
reference_audio_urls: some(urlsOf('audio')),
|
|
1248
|
+
duration, aspect_ratio, motion, preset_id, enhance_prompt, visual_dna_ids,
|
|
1249
|
+
resolution, sound_enabled, keyframes, multi_shots, multi_shot_count,
|
|
1250
|
+
session_name, project_id, session_id
|
|
1251
|
+
};
|
|
1252
|
+
|
|
1253
|
+
const startElements = async () => {
|
|
1254
|
+
if (!locals.length) {
|
|
1255
|
+
return client.post('/v1/generate/elements', body);
|
|
1256
|
+
}
|
|
1257
|
+
const resolved = await Promise.all(locals.map(({ src, kind }) =>
|
|
1258
|
+
resolveToBuffer(src, kind, { maxBytes: ELEMENTS_MAX_UPLOAD_BYTES })));
|
|
1199
1259
|
const form = new FormData();
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
if (session_name) form.append('session_name', session_name);
|
|
1218
|
-
if (project_id) form.append('project_id', project_id);
|
|
1219
|
-
if (session_id) form.append('session_id', session_id);
|
|
1260
|
+
for (const [key, value] of Object.entries(body)) {
|
|
1261
|
+
if (value === undefined || value === null) continue;
|
|
1262
|
+
// The two URL lists are read by DIFFERENT parsers on the API side and
|
|
1263
|
+
// only one shape satisfies both. reference_videos/_audio_urls go
|
|
1264
|
+
// through parseJsonArray, which needs a JSON string (a bare url makes
|
|
1265
|
+
// JSON.parse throw and the list is dropped). reference_images goes
|
|
1266
|
+
// through collectHttpMediaUrls, which walks strings and arrays but
|
|
1267
|
+
// never JSON-decodes — so a stringified array fails its
|
|
1268
|
+
// `^https?://` test and every URL reference is silently lost. Repeated
|
|
1269
|
+
// form fields arrive as a string (one) or an array (several), and that
|
|
1270
|
+
// walker handles both.
|
|
1271
|
+
if (key === 'reference_images') {
|
|
1272
|
+
for (const url of value) form.append('reference_images', url);
|
|
1273
|
+
continue;
|
|
1274
|
+
}
|
|
1275
|
+
form.append(key, typeof value === 'object' ? JSON.stringify(value) : String(value));
|
|
1276
|
+
}
|
|
1220
1277
|
for (const f of resolved) {
|
|
1221
1278
|
form.append('files', f.buffer, { filename: f.filename, contentType: f.contentType });
|
|
1222
1279
|
}
|
|
1223
|
-
|
|
1224
|
-
}
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1280
|
+
return client.postMultipart('/v1/generate/elements', form);
|
|
1281
|
+
};
|
|
1282
|
+
|
|
1283
|
+
let startResponse;
|
|
1284
|
+
try {
|
|
1285
|
+
startResponse = await startElements();
|
|
1286
|
+
} catch (err) {
|
|
1287
|
+
// The elements trust gate only accepts remote references that are BOTH
|
|
1288
|
+
// Kolbo-hosted AND registered to this caller's library/project — any
|
|
1289
|
+
// other public URL 400s with this code, even though every schema here
|
|
1290
|
+
// says "public URL, forwarded as-is". Agents used to recover by hand
|
|
1291
|
+
// (upload_media, then resend). Do that detour for them: rehost every
|
|
1292
|
+
// remote reference into the caller's library and retry ONCE. The gate
|
|
1293
|
+
// fires before any charge, so the failed first attempt costs nothing.
|
|
1294
|
+
if (err?.code !== 'UNTRUSTED_REFERENCE_MEDIA_URL') throw err;
|
|
1295
|
+
for (const kind of ['image', 'video', 'audio']) {
|
|
1296
|
+
media[kind] = await Promise.all(media[kind].map(async (src) => {
|
|
1297
|
+
if (!isUrlSource(src)) return src;
|
|
1298
|
+
const file = await resolveToBuffer(src, kind, { maxBytes: ELEMENTS_MAX_UPLOAD_BYTES });
|
|
1299
|
+
const form = new FormData();
|
|
1300
|
+
form.append('file', file.buffer, { filename: file.filename, contentType: file.contentType });
|
|
1301
|
+
if (project_id) form.append('project_id', project_id);
|
|
1302
|
+
const uploaded = await client.postMultipart('/v1/media/upload', form);
|
|
1303
|
+
return uploaded?.media?.url || uploaded?.url || src;
|
|
1304
|
+
}));
|
|
1305
|
+
}
|
|
1306
|
+
body.reference_images = some(urlsOf('image'));
|
|
1307
|
+
body.reference_videos = some(urlsOf('video'));
|
|
1308
|
+
body.reference_audio_urls = some(urlsOf('audio'));
|
|
1309
|
+
startResponse = await startElements();
|
|
1229
1310
|
}
|
|
1230
1311
|
|
|
1231
1312
|
if (ui()) return uiGenerating({
|
|
@@ -1236,16 +1317,11 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1236
1317
|
shots: multi_shot_count ?? (Array.isArray(multi_shots) ? multi_shots.length : undefined),
|
|
1237
1318
|
resolution, aspect_ratio, enhance_prompt, visual_dna_ids, preset_id,
|
|
1238
1319
|
}),
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
// Elements is the one tool that takes all three modalities. The widget
|
|
1245
|
-
// sorts kind by extension, so a video that arrived via `files` is still
|
|
1246
|
-
// rendered as a video — these two just make sure nothing is dropped.
|
|
1247
|
-
reference_videos: reference_videos || [],
|
|
1248
|
-
reference_audio: [...(reference_audio_urls || []), ...(audio_url ? [audio_url] : [])]
|
|
1320
|
+
// Already bucketed and deduped above, so a URL handed to `files`
|
|
1321
|
+
// renders under its real kind instead of being dropped or double-listed.
|
|
1322
|
+
reference_images: [...urlsOf('image'), ...(keyframes || []).map((keyframe) => keyframe.image_url)],
|
|
1323
|
+
reference_videos: urlsOf('video'),
|
|
1324
|
+
reference_audio: urlsOf('audio')
|
|
1249
1325
|
});
|
|
1250
1326
|
|
|
1251
1327
|
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
|
|
@@ -1263,13 +1339,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1263
1339
|
shots: multi_shot_count ?? (Array.isArray(multi_shots) ? multi_shots.length : undefined),
|
|
1264
1340
|
resolution, aspect_ratio, enhance_prompt, visual_dna_ids, preset_id,
|
|
1265
1341
|
}),
|
|
1266
|
-
reference_images: [
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
...(files || []).filter((source) => /^https?:\/\//i.test(source))
|
|
1270
|
-
],
|
|
1271
|
-
reference_videos: reference_videos || [],
|
|
1272
|
-
reference_audio: [...(reference_audio_urls || []), ...(audio_url ? [audio_url] : [])],
|
|
1342
|
+
reference_images: [...urlsOf('image'), ...(keyframes || []).map((keyframe) => keyframe.image_url)],
|
|
1343
|
+
reference_videos: urlsOf('video'),
|
|
1344
|
+
reference_audio: urlsOf('audio'),
|
|
1273
1345
|
urls: result.result?.urls || [],
|
|
1274
1346
|
thumbnail_url: result.result?.thumbnail_url || null,
|
|
1275
1347
|
duration: result.result?.duration || null,
|
|
@@ -1288,12 +1360,12 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1288
1360
|
// ─── generate_first_last_frame ─────────────────────────────
|
|
1289
1361
|
server.tool(
|
|
1290
1362
|
'generate_first_last_frame',
|
|
1291
|
-
'Generate a video that morphs / interpolates from a FIRST frame to a LAST frame. Provide the two frames as URLs (first_frame_url + last_frame_url)
|
|
1363
|
+
'Generate a video that morphs / interpolates from a FIRST frame to a LAST frame. Provide the two frames as URLs (first_frame_url + last_frame_url) or as URLs/absolute local paths (first_frame + last_frame) — the pairs are interchangeable and may be mixed. Optional prompt describes the desired motion/transition. Returns the final video URL when complete.',
|
|
1292
1364
|
{
|
|
1293
|
-
first_frame_url: z.string().optional().describe('Public URL of the first frame image
|
|
1294
|
-
last_frame_url: z.string().optional().describe('Public URL of the last frame image
|
|
1295
|
-
first_frame: z.string().optional().describe('URL or absolute local path to the first frame
|
|
1296
|
-
last_frame: z.string().optional().describe('URL or absolute local path to the last frame
|
|
1365
|
+
first_frame_url: z.string().optional().describe('Public URL of the first frame image. Interchangeable with first_frame.'),
|
|
1366
|
+
last_frame_url: z.string().optional().describe('Public URL of the last frame image. Interchangeable with last_frame.'),
|
|
1367
|
+
first_frame: z.string().optional().describe('URL or absolute local path to the first frame — alternative to first_frame_url. A URL here is forwarded as a URL, not re-uploaded.'),
|
|
1368
|
+
last_frame: z.string().optional().describe('URL or absolute local path to the last frame — alternative to last_frame_url. A URL here is forwarded as a URL, not re-uploaded.'),
|
|
1297
1369
|
prompt: z.string().optional().describe('Optional description of the desired motion between the two frames (e.g. "smooth camera dolly in")'),
|
|
1298
1370
|
model: z.string().optional().describe('Model identifier. Use list_models type="firstlastgenerations" to see options. Pick a SPECIFIC model — do NOT omit (omitting = Smart Select auto-pick, which we avoid); call list_models for this type and choose the model that best fits the user\'s intent.'),
|
|
1299
1371
|
duration: z.number().optional().describe('Duration in seconds. Must be in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. Default: 5'),
|
|
@@ -1307,20 +1379,24 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1307
1379
|
},
|
|
1308
1380
|
async ({ first_frame_url, last_frame_url, first_frame, last_frame, prompt, model, duration, aspect_ratio, enhance_prompt = false, visual_dna_ids, resolution, sound_enabled, project_id, session_id }) => {
|
|
1309
1381
|
model = await canonicalModelId(client, model, 'firstlastgenerations'); // lenient id resolution ("z-image" → "z-image/turbo")
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1382
|
+
// One frame per position, whichever arg carried it. The two arg pairs were
|
|
1383
|
+
// treated as two exclusive MODES, so a URL handed to first_frame/last_frame
|
|
1384
|
+
// (which their own descriptions invite) took the multipart path and got
|
|
1385
|
+
// downloaded and re-uploaded — a pointless round-trip that also had to fit
|
|
1386
|
+
// the route's 10MB upload ceiling a plain URL never touches. Transport is
|
|
1387
|
+
// now decided per frame by what the value IS, not which arg it arrived in,
|
|
1388
|
+
// which also makes one-URL-one-local work instead of erroring out.
|
|
1389
|
+
const firstSource = first_frame_url || first_frame;
|
|
1390
|
+
const lastSource = last_frame_url || last_frame;
|
|
1391
|
+
if (!firstSource || !lastSource) {
|
|
1392
|
+
throw new Error('Provide both frames: first_frame_url + last_frame_url (URLs) or first_frame + last_frame (URL or absolute local path).');
|
|
1317
1393
|
}
|
|
1318
1394
|
|
|
1319
1395
|
let startResponse;
|
|
1320
|
-
if (
|
|
1396
|
+
if (!isUrlSource(firstSource) || !isUrlSource(lastSource)) {
|
|
1321
1397
|
const [firstResolved, lastResolved] = await Promise.all([
|
|
1322
|
-
resolveToBuffer(
|
|
1323
|
-
resolveToBuffer(
|
|
1398
|
+
resolveToBuffer(firstSource, 'image'),
|
|
1399
|
+
resolveToBuffer(lastSource, 'image')
|
|
1324
1400
|
]);
|
|
1325
1401
|
const form = new FormData();
|
|
1326
1402
|
form.append('files', firstResolved.buffer, { filename: firstResolved.filename, contentType: firstResolved.contentType });
|
|
@@ -1338,15 +1414,15 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1338
1414
|
startResponse = await client.postMultipart('/v1/generate/first-last-frame', form);
|
|
1339
1415
|
} else {
|
|
1340
1416
|
startResponse = await client.post('/v1/generate/first-last-frame', {
|
|
1341
|
-
first_frame_url, last_frame_url
|
|
1417
|
+
first_frame_url: firstSource, last_frame_url: lastSource,
|
|
1418
|
+
prompt, model, duration, aspect_ratio, enhance_prompt, visual_dna_ids, resolution, sound_enabled, project_id, session_id
|
|
1342
1419
|
});
|
|
1343
1420
|
}
|
|
1344
1421
|
|
|
1345
1422
|
if (ui()) return uiGenerating({
|
|
1346
1423
|
tool: 'generate_first_last_frame', kind: 'video', gen: startResponse, client, model, prompt,
|
|
1347
1424
|
settings: videoSettings({ duration, resolution, aspect_ratio, enhance_prompt, visual_dna_ids }),
|
|
1348
|
-
reference_images: [
|
|
1349
|
-
.filter((source) => /^https?:\/\//i.test(source || ''))
|
|
1425
|
+
reference_images: [firstSource, lastSource].filter(isUrlSource)
|
|
1350
1426
|
});
|
|
1351
1427
|
|
|
1352
1428
|
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
|
|
@@ -1359,8 +1435,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1359
1435
|
return uiCompleted({
|
|
1360
1436
|
tool: 'generate_first_last_frame', kind: 'video', gen: startResponse, client, model, prompt,
|
|
1361
1437
|
settings: videoSettings({ duration, resolution, aspect_ratio, enhance_prompt, visual_dna_ids }),
|
|
1362
|
-
reference_images: [
|
|
1363
|
-
.filter((source) => /^https?:\/\//i.test(source || '')),
|
|
1438
|
+
reference_images: [firstSource, lastSource].filter(isUrlSource),
|
|
1364
1439
|
urls: result.result?.urls || [],
|
|
1365
1440
|
thumbnail_url: result.result?.thumbnail_url || null,
|
|
1366
1441
|
duration: result.result?.duration || null,
|
|
@@ -1530,10 +1605,18 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1530
1605
|
highlighted: z.object({ font: z.string().optional(), weight: z.number().int().min(100).max(900).optional(), color: z.string().optional() }).optional().describe('Highlighted word tier styling.'),
|
|
1531
1606
|
}).optional(),
|
|
1532
1607
|
}).optional().describe('VEED Subtitles only: style overrides. Any omitted field keeps the preset default. Best supported by Basic presets.'),
|
|
1608
|
+
enhancement_model: z.string().optional()
|
|
1609
|
+
.describe('Topaz Slow Motion only (model "topaz/interpolate/video"): retiming engine — "Apollo" (smooth motion, default), "Chronos" (complex motion and occlusion), or "Aion" (highest quality for extreme slow motion, and the most expensive).'),
|
|
1610
|
+
target_fps: z.number().optional()
|
|
1611
|
+
.describe('Topaz Slow Motion only: frames per second of the OUTPUT (16-120, default 60). Higher rates generate more frames and cost proportionally more.'),
|
|
1612
|
+
slowdown_factor: z.number().optional()
|
|
1613
|
+
.describe('Topaz Slow Motion only: how many times longer the output runs (1-8, default 1). 4 turns a 5s clip into 20s of slow motion. Billing is on the OUTPUT length, so an 8x pass costs 8x a 1x pass.'),
|
|
1614
|
+
output_format: z.string().optional()
|
|
1615
|
+
.describe('Topaz HDR only (model "topaz/sdr-to-hdr/video"): "mp4" for 10-bit H.265 HDR10 (default) or "prores" for 10-bit ProRes 422 HQ. Both are HDR masters — the in-app player shows a tone-mapped SDR preview and the HDR file is the download.'),
|
|
1533
1616
|
project_id: projectIdField,
|
|
1534
1617
|
session_id: sessionIdField
|
|
1535
1618
|
},
|
|
1536
|
-
async ({ source_video, prompt, model, aspect_ratio, duration, enhance_prompt = false, visual_dna_ids, resolution, sound_enabled, reference_images, reference_videos, elements, preset, source_language, translation_language, srt_content, srt_file_url, vocabulary, customization, project_id, session_id }) => {
|
|
1619
|
+
async ({ source_video, prompt, model, aspect_ratio, duration, enhance_prompt = false, visual_dna_ids, resolution, sound_enabled, reference_images, reference_videos, elements, preset, source_language, translation_language, srt_content, srt_file_url, vocabulary, customization, enhancement_model, target_fps, slowdown_factor, output_format, project_id, session_id }) => {
|
|
1537
1620
|
model = await canonicalModelId(client, model, 'video_to_video'); // lenient id resolution ("z-image" → "z-image/turbo")
|
|
1538
1621
|
if (!source_video) throw new Error('source_video is required');
|
|
1539
1622
|
|
|
@@ -1543,7 +1626,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1543
1626
|
startResponse = await client.post('/v1/generate/video-from-video', {
|
|
1544
1627
|
video_url: source_video, prompt, model, aspect_ratio, duration, enhance_prompt, visual_dna_ids, resolution, sound_enabled,
|
|
1545
1628
|
reference_images, reference_videos, elements, preset, source_language, translation_language,
|
|
1546
|
-
srt_content, srt_file_url, vocabulary, customization,
|
|
1629
|
+
srt_content, srt_file_url, vocabulary, customization,
|
|
1630
|
+
enhancement_model, target_fps, slowdown_factor, output_format,
|
|
1631
|
+
project_id, session_id
|
|
1547
1632
|
});
|
|
1548
1633
|
} else {
|
|
1549
1634
|
const resolved = await resolveToBuffer(source_video, 'video');
|
|
@@ -1567,6 +1652,10 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1567
1652
|
if (reference_images) form.append('reference_images', JSON.stringify(reference_images));
|
|
1568
1653
|
if (reference_videos) form.append('reference_videos', JSON.stringify(reference_videos));
|
|
1569
1654
|
if (elements) form.append('elements', JSON.stringify(elements));
|
|
1655
|
+
if (enhancement_model) form.append('enhancement_model', enhancement_model);
|
|
1656
|
+
if (target_fps !== undefined) form.append('target_fps', String(target_fps));
|
|
1657
|
+
if (slowdown_factor !== undefined) form.append('slowdown_factor', String(slowdown_factor));
|
|
1658
|
+
if (output_format) form.append('output_format', output_format);
|
|
1570
1659
|
if (project_id) form.append('project_id', project_id);
|
|
1571
1660
|
if (session_id) form.append('session_id', session_id);
|
|
1572
1661
|
startResponse = await client.postMultipart('/v1/generate/video-from-video', form);
|
|
@@ -1776,7 +1865,8 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1776
1865
|
'camera_angle',
|
|
1777
1866
|
'split', 'split_upscale',
|
|
1778
1867
|
'multi_shot',
|
|
1779
|
-
'magic_edit'
|
|
1868
|
+
'magic_edit',
|
|
1869
|
+
'enhance'
|
|
1780
1870
|
]).describe([
|
|
1781
1871
|
'Edit operation:',
|
|
1782
1872
|
'"upscale" — increase resolution by 2×, 3×, or 4× (use `scale`). "clarity_upscale" — AI-powered clarity upscale with detail enhancement (use `resolution`).',
|
|
@@ -1785,6 +1875,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1785
1875
|
'"removebg" — remove the image background, output is transparent PNG.',
|
|
1786
1876
|
'"background_replace" — remove background and replace it with AI-generated content from `prompt`.',
|
|
1787
1877
|
'"enhance_skin" — portrait skin retouching (use `skin_strength`: "subtle" | "realistic" | "pimple" | "freckle").',
|
|
1878
|
+
'"enhance" — Topaz photo correction at the SOURCE resolution (no resizing). Pick the tool with `model`: "topaz/adjust/image" (exposure, white balance, or colorizing a black-and-white photo), "topaz/sharpen/image" (lens / motion / portrait / wildlife blur, or Super Focus for severely blurred shots), "topaz/denoise/image" (high-ISO and night noise), "topaz/restore/image" (old or damaged photos, dust and scratches). Choose the specific engine with `enhancement_model` — call list_models type="graphics_enhance" to see each model\'s engines. To make an image BIGGER use "upscale" instead.',
|
|
1788
1879
|
'"inpaint" — paint over a masked area using `mask_image_url` (B&W mask, white = fill area) and optional `prompt`. Add reference images via `additional_images`.',
|
|
1789
1880
|
'"erase" — erase an object defined by `mask_image_url` (white = erase area).',
|
|
1790
1881
|
'"face_swap" — swap the face in `image_url` with the face from `mask_image_url` (required).',
|
|
@@ -1799,7 +1890,13 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1799
1890
|
|
|
1800
1891
|
// ── upscale ────────────────────────────────────────────
|
|
1801
1892
|
scale: z.number().optional()
|
|
1802
|
-
.describe('Upscale factor:
|
|
1893
|
+
.describe('Upscale factor: 1, 2, 4 or 8. Used with operation="upscale". Default: 2.'),
|
|
1894
|
+
|
|
1895
|
+
enhancement_model: z.string().optional()
|
|
1896
|
+
.describe('Topaz engine. With operation="upscale" on model "topaz/upscale/image" it selects the engine family: "Standard V2" / "High Fidelity V3" / "CGI" / "Text Refine" (faithful), "Wonder 3.5" (rebuilds natural detail), "Bloom 2" (reinvents detail — most expensive), "Transparent" (keeps the alpha channel). With operation="enhance" it selects the correction engine for the chosen model. Omit for the model default. Call list_models to see the engines a model offers.'),
|
|
1897
|
+
|
|
1898
|
+
output_format: z.string().optional()
|
|
1899
|
+
.describe('Output image format: "jpeg" or "png". Used with Topaz "upscale" and "enhance". Defaults to jpeg (the transparent upscaler always returns png).'),
|
|
1803
1900
|
|
|
1804
1901
|
resolution: z.string().optional()
|
|
1805
1902
|
.describe('Target output resolution (e.g. "4k", "2k", "1080p"). Used with "clarity_upscale", "split_upscale", "multi_shot".'),
|
|
@@ -1856,6 +1953,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1856
1953
|
image_url, operation, model, scale, aspect_ratio, skin_strength, prompt,
|
|
1857
1954
|
mask_image_url, additional_images, generate_all_angles, resolution, quality, ai_optimize = false,
|
|
1858
1955
|
zoom_out_percentage, expand_left, expand_right, expand_top, expand_bottom,
|
|
1956
|
+
enhancement_model, output_format,
|
|
1859
1957
|
project_id, session_id
|
|
1860
1958
|
}) => {
|
|
1861
1959
|
// No `type` argument: these are operation-routed tools (upscale / reframe /
|
|
@@ -1865,6 +1963,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1865
1963
|
|
|
1866
1964
|
// Basic validation
|
|
1867
1965
|
if (operation === 'reframe' && !aspect_ratio) throw new Error('aspect_ratio is required for reframe');
|
|
1966
|
+
if (operation === 'enhance' && !model) {
|
|
1967
|
+
throw new Error('model is required for enhance — pick one of: topaz/adjust/image, topaz/sharpen/image, topaz/denoise/image, topaz/restore/image');
|
|
1968
|
+
}
|
|
1868
1969
|
if (operation === 'background_replace' && !prompt) throw new Error('prompt is required for background_replace');
|
|
1869
1970
|
if (operation === 'face_swap' && !mask_image_url && !(additional_images && additional_images.length > 0)) {
|
|
1870
1971
|
throw new Error('mask_image_url (face reference) is required for face_swap');
|
|
@@ -1874,6 +1975,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1874
1975
|
image_url, operation, model, scale, aspect_ratio, skin_strength, prompt,
|
|
1875
1976
|
mask_image_url, additional_images, generate_all_angles, resolution, quality, ai_optimize,
|
|
1876
1977
|
zoom_out_percentage, expand_left, expand_right, expand_top, expand_bottom,
|
|
1978
|
+
enhancement_model, output_format,
|
|
1877
1979
|
project_id, session_id
|
|
1878
1980
|
});
|
|
1879
1981
|
|
|
@@ -1946,6 +2048,8 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1946
2048
|
.describe('Target resolution (e.g. "4k", "2k", "1080p"). Used with "upscale" and "reframe".'),
|
|
1947
2049
|
target_fps: z.number().optional()
|
|
1948
2050
|
.describe('Target frame rate (e.g. 24, 30, 60). Used with operation="upscale".'),
|
|
2051
|
+
enhancement_model: z.string().optional()
|
|
2052
|
+
.describe('Topaz enhancement engine for operation="upscale" on model "topaz/upscale/video". Families: Precision (faithful — "Proteus", "Artemis High Quality", "Iris", "Dione TV", "Gaia CG", "Gaia 2"), Denoise ("Nyx", "Nyx Fast"), Generative (rebuilds detail that is not in the source — "Starlight Fast 2", "Starlight HQ", "Starlight Precise 2.6"), Creative ("Astra 2", always renders 4K). Generative and Creative engines cost several times the Precision rate. Omit for the default.'),
|
|
1949
2053
|
|
|
1950
2054
|
// ── reframe ────────────────────────────────────────────
|
|
1951
2055
|
aspect_ratio: z.string().optional()
|
|
@@ -2009,7 +2113,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
2009
2113
|
async ({
|
|
2010
2114
|
video_url, operation, model, aspect_ratio, scale, prompt,
|
|
2011
2115
|
image_url, audio_url, duration, mode,
|
|
2012
|
-
target_fps, resolution,
|
|
2116
|
+
target_fps, resolution, enhancement_model,
|
|
2013
2117
|
grid_position_x, grid_position_y,
|
|
2014
2118
|
sound_effect_prompt, background_music_prompt, original_sound, cfg_strength,
|
|
2015
2119
|
refine_edges, subject_is_person,
|
|
@@ -2032,7 +2136,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
2032
2136
|
if (operation === 'extend' && !duration) throw new Error('duration is required for extend');
|
|
2033
2137
|
|
|
2034
2138
|
const gen = await client.post('/v1/edit/video', {
|
|
2035
|
-
video_url, operation, model, aspect_ratio, scale, prompt,
|
|
2139
|
+
video_url, operation, model, aspect_ratio, scale, prompt, enhancement_model,
|
|
2036
2140
|
image_url, audio_url, duration, mode,
|
|
2037
2141
|
target_fps, resolution,
|
|
2038
2142
|
grid_position_x, grid_position_y,
|