@kolbo/mcp 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/package.json +9 -3
- package/src/index.js +4 -0
- package/src/polling.js +1 -1
- package/src/tools/_shared.js +212 -0
- package/src/tools/chat.js +13 -12
- package/src/tools/generate.js +415 -62
- package/src/tools/media.js +78 -0
- package/src/tools/models.js +52 -53
- package/src/tools/moodboards.js +3 -1
- package/src/tools/presets.js +36 -0
- package/src/tools/visual_dna.js +15 -78
package/src/tools/generate.js
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
* `npx @kolbo/mcp` installs in the wild will break silently. Add new tools or
|
|
4
4
|
* new OPTIONAL args only. Full rules: ../index.js top-of-file and CLAUDE.md. */
|
|
5
5
|
|
|
6
|
+
const { z } = require('zod');
|
|
7
|
+
const FormData = require('form-data');
|
|
6
8
|
const { pollUntilDone } = require('../polling');
|
|
9
|
+
const { resolveToBuffer } = require('./_shared');
|
|
7
10
|
|
|
8
11
|
function registerGenerateTools(server, client) {
|
|
9
12
|
// ─── generate_image ────────────────────────────────────────
|
|
@@ -11,15 +14,15 @@ function registerGenerateTools(server, client) {
|
|
|
11
14
|
'generate_image',
|
|
12
15
|
'Generate image(s) from a text prompt using Kolbo AI. Supports Visual DNA profiles (for character/style/product consistency), moodboards (for style direction), reference images (for composition guidance), batch generation (num_images), and web-search grounding. For EDITING an existing image, use generate_image_edit instead. For a coordinated multi-scene set (storyboard, ad campaign), use generate_creative_director. Returns the final image URL(s) when complete.',
|
|
13
16
|
{
|
|
14
|
-
prompt:
|
|
15
|
-
model:
|
|
16
|
-
aspect_ratio:
|
|
17
|
-
enhance_prompt:
|
|
18
|
-
num_images:
|
|
19
|
-
reference_images:
|
|
20
|
-
visual_dna_ids:
|
|
21
|
-
moodboard_id:
|
|
22
|
-
enable_web_search:
|
|
17
|
+
prompt: z.string().describe('Text description of the image to generate'),
|
|
18
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="image" to see options. Omit for Smart Select.'),
|
|
19
|
+
aspect_ratio: z.string().optional().describe('Aspect ratio (e.g., "1:1", "16:9", "9:16"). Default: "1:1"'),
|
|
20
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt for better results. Default: true'),
|
|
21
|
+
num_images: z.number().optional().describe('Number of images to generate in one call. Default: 1'),
|
|
22
|
+
reference_images: z.array(z.string()).optional().describe('Array of image URLs used as composition/style references (NOT as source images for editing — use generate_image_edit for that).'),
|
|
23
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs (from create_visual_dna / list_visual_dnas) to apply for character / style / product / scene consistency. Pass the `id` field of each profile. Use this when the user wants to keep the same character or style across multiple images.'),
|
|
24
|
+
moodboard_id: z.string().optional().describe('Moodboard ID (from list_moodboards / get_moodboard) whose master_prompt and style_guide should be applied to this generation.'),
|
|
25
|
+
enable_web_search: z.boolean().optional().describe('Enable web-search grounding for the prompt (useful for current events, brand references, real-world accuracy). Default: false')
|
|
23
26
|
},
|
|
24
27
|
async ({ prompt, model, aspect_ratio, enhance_prompt, num_images, reference_images, visual_dna_ids, moodboard_id, enable_web_search }) => {
|
|
25
28
|
const gen = await client.post('/v1/generate/image', {
|
|
@@ -50,15 +53,15 @@ function registerGenerateTools(server, client) {
|
|
|
50
53
|
'generate_image_edit',
|
|
51
54
|
'Edit or transform an existing image using AI. Provide the source image URL(s) in `source_images` and describe the edit in `prompt` (e.g., "remove the background", "change the car color to red", "add sunglasses to the person"). Supports Visual DNA profiles and moodboards for style-consistent edits. For creating a brand new image from scratch, use generate_image. Returns the edited image URL(s) when complete.',
|
|
52
55
|
{
|
|
53
|
-
prompt:
|
|
54
|
-
model:
|
|
55
|
-
source_images:
|
|
56
|
-
aspect_ratio:
|
|
57
|
-
enhance_prompt:
|
|
58
|
-
num_images:
|
|
59
|
-
visual_dna_ids:
|
|
60
|
-
moodboard_id:
|
|
61
|
-
enable_web_search:
|
|
56
|
+
prompt: z.string().describe('Description of the edit to apply (e.g., "remove the background", "change the sky to sunset")'),
|
|
57
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="image_edit" to see options. Omit for Smart Select.'),
|
|
58
|
+
source_images: z.array(z.string()).describe('Array of source image URLs to edit. Typically one, but some models accept multiple for compositing.'),
|
|
59
|
+
aspect_ratio: z.string().optional().describe('Output aspect ratio (e.g., "1:1", "16:9", "9:16"). Default: "1:1"'),
|
|
60
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt for better results. Default: true'),
|
|
61
|
+
num_images: z.number().optional().describe('Number of output images. Default: 1'),
|
|
62
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for consistency with an existing character / style / product.'),
|
|
63
|
+
moodboard_id: z.string().optional().describe('Moodboard ID whose master_prompt and style_guide should be applied.'),
|
|
64
|
+
enable_web_search: z.boolean().optional().describe('Enable web-search grounding. Default: false')
|
|
62
65
|
},
|
|
63
66
|
async ({ prompt, model, source_images, aspect_ratio, enhance_prompt, num_images, visual_dna_ids, moodboard_id, enable_web_search }) => {
|
|
64
67
|
const gen = await client.post('/v1/generate/image-edit', {
|
|
@@ -89,17 +92,17 @@ function registerGenerateTools(server, client) {
|
|
|
89
92
|
'generate_creative_director',
|
|
90
93
|
'Generate a multi-scene coordinated set from ONE creative brief. Use this INSTEAD of calling generate_image/generate_video multiple times when the user wants a storyboard, multi-scene ad, product showcase, or any set of related outputs that should share visual language. Produces 1–8 scenes in a single request with consistent style. Supports image mode and video mode (`workflow_type`). Visual DNA and moodboard references keep character/style consistent across every scene.',
|
|
91
94
|
{
|
|
92
|
-
prompt:
|
|
93
|
-
scene_count:
|
|
94
|
-
model:
|
|
95
|
-
aspect_ratio:
|
|
96
|
-
workflow_type:
|
|
97
|
-
duration:
|
|
98
|
-
enhance_prompt:
|
|
99
|
-
reference_images:
|
|
100
|
-
visual_dna_ids:
|
|
101
|
-
moodboard_id:
|
|
102
|
-
moodboard_ids:
|
|
95
|
+
prompt: z.string().describe('Creative brief or concept describing the full set of scenes to generate'),
|
|
96
|
+
scene_count: z.number().optional().describe('Number of scenes to generate, 1–8. Default: 4'),
|
|
97
|
+
model: z.string().optional().describe('Model identifier applied to every scene. Omit for Smart Select.'),
|
|
98
|
+
aspect_ratio: z.string().optional().describe('Aspect ratio applied to every scene (e.g., "1:1", "16:9", "9:16"). Default: "1:1"'),
|
|
99
|
+
workflow_type: z.string().optional().describe('"image" (default) or "video"'),
|
|
100
|
+
duration: z.number().optional().describe('Duration in seconds per scene (video mode only). E.g., 5 or 10.'),
|
|
101
|
+
enhance_prompt: z.boolean().optional().describe('Enhance prompts per scene. Default: true'),
|
|
102
|
+
reference_images: z.array(z.string()).optional().describe('Array of reference image URLs to guide style/composition of every scene.'),
|
|
103
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply consistently across every scene. This is the ideal way to keep a character or product looking the same in all scenes of a campaign.'),
|
|
104
|
+
moodboard_id: z.string().optional().describe('A single moodboard ID whose master_prompt and style_guide should shape every scene.'),
|
|
105
|
+
moodboard_ids: z.array(z.string()).optional().describe('Multiple moodboard IDs when blending styles. Prefer `moodboard_id` for single moodboards.')
|
|
103
106
|
},
|
|
104
107
|
async ({ prompt, scene_count, model, aspect_ratio, workflow_type, duration, enhance_prompt, reference_images, visual_dna_ids, moodboard_id, moodboard_ids }) => {
|
|
105
108
|
const gen = await client.post('/v1/generate/creative-director', {
|
|
@@ -140,13 +143,13 @@ function registerGenerateTools(server, client) {
|
|
|
140
143
|
'generate_video',
|
|
141
144
|
'Generate a video from a text prompt using Kolbo AI. For animating an existing still image into motion, use generate_video_from_image instead. For a coordinated multi-scene video campaign, use generate_creative_director with workflow_type="video". Supports Visual DNA profiles (for character consistency) and reference images (for style guidance). Returns the final video URL when complete.',
|
|
142
145
|
{
|
|
143
|
-
prompt:
|
|
144
|
-
model:
|
|
145
|
-
aspect_ratio:
|
|
146
|
-
duration:
|
|
147
|
-
enhance_prompt:
|
|
148
|
-
reference_images:
|
|
149
|
-
visual_dna_ids:
|
|
146
|
+
prompt: z.string().describe('Text description of the video to generate'),
|
|
147
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="video" to see options. Check supported_durations and supported_aspect_ratios.'),
|
|
148
|
+
aspect_ratio: z.string().optional().describe('Aspect ratio (e.g., "16:9", "9:16", "1:1"). Default: "16:9"'),
|
|
149
|
+
duration: z.number().optional().describe('Duration in seconds. Must be a value the chosen model supports — check supported_durations from list_models. Default: 5'),
|
|
150
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: true'),
|
|
151
|
+
reference_images: z.array(z.string()).optional().describe('Array of image URLs used as visual references (style / composition / subject).'),
|
|
152
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to keep a character / style consistent with prior generations.')
|
|
150
153
|
},
|
|
151
154
|
async ({ prompt, model, aspect_ratio, duration, enhance_prompt, reference_images, visual_dna_ids }) => {
|
|
152
155
|
const gen = await client.post('/v1/generate/video', {
|
|
@@ -178,13 +181,13 @@ function registerGenerateTools(server, client) {
|
|
|
178
181
|
'generate_video_from_image',
|
|
179
182
|
'Animate an existing still image into a video using Kolbo AI. The image comes from `image_url`; `prompt` describes the motion (not the subject — the subject is already in the image). For generating a video from scratch, use generate_video. Returns the final video URL when complete.',
|
|
180
183
|
{
|
|
181
|
-
image_url:
|
|
182
|
-
prompt:
|
|
183
|
-
model:
|
|
184
|
-
aspect_ratio:
|
|
185
|
-
duration:
|
|
186
|
-
enhance_prompt:
|
|
187
|
-
visual_dna_ids:
|
|
184
|
+
image_url: z.string().describe('URL of the source image to animate'),
|
|
185
|
+
prompt: z.string().describe('Text description of the desired MOTION (e.g., "camera slowly pans right while the character walks forward")'),
|
|
186
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="video_from_image" to see options.'),
|
|
187
|
+
aspect_ratio: z.string().optional().describe('Output aspect ratio (e.g., "16:9", "9:16", "1:1"). Default: "16:9"'),
|
|
188
|
+
duration: z.number().optional().describe('Duration in seconds. Must be a value the chosen model supports. Default: 5'),
|
|
189
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the motion prompt. Default: true'),
|
|
190
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to maintain consistency with prior characters / styles.')
|
|
188
191
|
},
|
|
189
192
|
async ({ image_url, prompt, model, aspect_ratio, duration, enhance_prompt, visual_dna_ids }) => {
|
|
190
193
|
const gen = await client.post('/v1/generate/video/from-image', {
|
|
@@ -215,13 +218,13 @@ function registerGenerateTools(server, client) {
|
|
|
215
218
|
'generate_music',
|
|
216
219
|
'Generate music from a text description using Kolbo AI. Supports instrumental mode, custom lyrics, style direction, and vocal gender. Default model is Suno. Returns the final audio URL when complete.',
|
|
217
220
|
{
|
|
218
|
-
prompt:
|
|
219
|
-
model:
|
|
220
|
-
style:
|
|
221
|
-
instrumental:
|
|
222
|
-
lyrics:
|
|
223
|
-
vocal_gender:
|
|
224
|
-
enhance_prompt:
|
|
221
|
+
prompt: z.string().describe('Text description of the music to generate (e.g., "upbeat electronic dance track with synthesizers")'),
|
|
222
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="music" to see options. Omit for Suno (default).'),
|
|
223
|
+
style: z.string().optional().describe('Music style / genre (e.g., "pop", "rock", "lo-fi", "electronic", "jazz")'),
|
|
224
|
+
instrumental: z.boolean().optional().describe('Generate instrumental only, no vocals. Default: false'),
|
|
225
|
+
lyrics: z.string().optional().describe('Custom lyrics for the song. If omitted, lyrics are generated automatically from the prompt unless instrumental is true.'),
|
|
226
|
+
vocal_gender: z.string().optional().describe('Preferred vocal gender: "male" or "female". Only applies when instrumental is false.'),
|
|
227
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: true')
|
|
225
228
|
},
|
|
226
229
|
async ({ prompt, model, style, instrumental, lyrics, vocal_gender, enhance_prompt }) => {
|
|
227
230
|
const gen = await client.post('/v1/generate/music', {
|
|
@@ -252,10 +255,10 @@ function registerGenerateTools(server, client) {
|
|
|
252
255
|
'generate_speech',
|
|
253
256
|
'Convert text to speech using Kolbo AI. Default provider is ElevenLabs. To pick a specific voice by language/gender, call list_voices first and pass the returned voice_id (or a voice display name — both work). Returns the final audio URL when complete.',
|
|
254
257
|
{
|
|
255
|
-
text:
|
|
256
|
-
voice:
|
|
257
|
-
model:
|
|
258
|
-
language:
|
|
258
|
+
text: z.string().describe('The text to convert to speech'),
|
|
259
|
+
voice: z.string().optional().describe('Voice ID (from list_voices) or voice display name (e.g., "Rachel", "Adam"). Default: "Rachel"'),
|
|
260
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="speech" to see options. Default: eleven_v3'),
|
|
261
|
+
language: z.string().optional().describe('Language code (e.g., "en-US", "he-IL", "es-ES"). Default: "en-US"')
|
|
259
262
|
},
|
|
260
263
|
async ({ text, voice, model, language }) => {
|
|
261
264
|
const gen = await client.post('/v1/generate/speech', {
|
|
@@ -285,9 +288,9 @@ function registerGenerateTools(server, client) {
|
|
|
285
288
|
'generate_sound',
|
|
286
289
|
'Generate sound effects (not music, not speech) from a text description using Kolbo AI. Use this for ambient sounds, foley, impacts, atmospheres, UI sounds, etc. For music use generate_music; for voice use generate_speech. Returns the final audio URL when complete.',
|
|
287
290
|
{
|
|
288
|
-
prompt:
|
|
289
|
-
model:
|
|
290
|
-
duration:
|
|
291
|
+
prompt: z.string().describe('Text description of the sound effect (e.g., "thunder clap with rain", "door creaking open", "futuristic UI beep")'),
|
|
292
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="sound" to see options. Default: elevenlabs-sound-effects-v1'),
|
|
293
|
+
duration: z.number().optional().describe('Duration in seconds. Omit for automatic duration.')
|
|
291
294
|
},
|
|
292
295
|
async ({ prompt, model, duration }) => {
|
|
293
296
|
const gen = await client.post('/v1/generate/sound', {
|
|
@@ -316,9 +319,9 @@ function registerGenerateTools(server, client) {
|
|
|
316
319
|
'list_voices',
|
|
317
320
|
'List available TTS voices for generate_speech. Returns preset voices and the user\'s own cloned/designed voices. Filter by provider, language, or gender to find the right voice. Use the returned `voice_id` as the `voice` parameter in generate_speech.',
|
|
318
321
|
{
|
|
319
|
-
provider:
|
|
320
|
-
language:
|
|
321
|
-
gender:
|
|
322
|
+
provider: z.string().optional().describe('Filter by provider (e.g., "elevenLabs", "google")'),
|
|
323
|
+
language: z.string().optional().describe('Filter by language name or code (e.g., "English", "en-US")'),
|
|
324
|
+
gender: z.string().optional().describe('Filter by gender (e.g., "Female", "Male")')
|
|
322
325
|
},
|
|
323
326
|
async ({ provider, language, gender }) => {
|
|
324
327
|
const params = new URLSearchParams();
|
|
@@ -353,10 +356,10 @@ function registerGenerateTools(server, client) {
|
|
|
353
356
|
'get_generation_status',
|
|
354
357
|
'Check the status of a generation. Use this as a FALLBACK when a generation tool returned a timeout error — the generation is probably still running on the server. Pass the generation_id from the timeout error (or from any prior generation response).',
|
|
355
358
|
{
|
|
356
|
-
generation_id:
|
|
359
|
+
generation_id: z.string().describe('The generation ID to check')
|
|
357
360
|
},
|
|
358
361
|
async ({ generation_id }) => {
|
|
359
|
-
const result = await client.get(`/v1/generate/${generation_id}/status`);
|
|
362
|
+
const result = await client.get(`/v1/generate/${encodeURIComponent(generation_id)}/status`);
|
|
360
363
|
|
|
361
364
|
return {
|
|
362
365
|
content: [{
|
|
@@ -366,6 +369,356 @@ function registerGenerateTools(server, client) {
|
|
|
366
369
|
};
|
|
367
370
|
}
|
|
368
371
|
);
|
|
372
|
+
|
|
373
|
+
// ═════════════════════════════════════════════════════════════
|
|
374
|
+
// ─── 2026-04 SDK Expansion Batch ─────────────────────────────
|
|
375
|
+
// ═════════════════════════════════════════════════════════════
|
|
376
|
+
|
|
377
|
+
// ─── generate_elements ─────────────────────────────────────
|
|
378
|
+
server.tool(
|
|
379
|
+
'generate_elements',
|
|
380
|
+
'Generate a video from reference elements (images and/or videos) + a text prompt. Use when the user wants to animate specific uploaded/referenced assets — e.g. "animate this product", "put these 3 characters into a scene". Supports Visual DNA for character consistency. For text-only → video use generate_video instead. For animating a single still image use generate_video_from_image. Returns the final video URL when complete.',
|
|
381
|
+
{
|
|
382
|
+
prompt: z.string().describe('Text description of the desired video / animation'),
|
|
383
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="video" to see options. Omit for Smart Select.'),
|
|
384
|
+
reference_images: z.array(z.string()).optional().describe('Array of public image URLs used as reference elements (product shots, character references, etc.). URL mode.'),
|
|
385
|
+
files: z.array(z.string()).optional().describe('Array of URLs or absolute local paths — alternative to reference_images. Use this when you have local files to upload. Each item can be a URL OR a local path.'),
|
|
386
|
+
duration: z.number().optional().describe('Duration in seconds. Default: 5'),
|
|
387
|
+
aspect_ratio: z.string().optional().describe('Aspect ratio (e.g., "16:9", "9:16", "1:1"). Default: "16:9"'),
|
|
388
|
+
motion: z.string().optional().describe('Motion style / intensity hint (optional)'),
|
|
389
|
+
preset_id: z.string().optional().describe('Preset ID from list_presets type="video" (optional)'),
|
|
390
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: true'),
|
|
391
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for character/style consistency across outputs.')
|
|
392
|
+
},
|
|
393
|
+
async ({ prompt, model, reference_images, files, duration, aspect_ratio, motion, preset_id, enhance_prompt, visual_dna_ids }) => {
|
|
394
|
+
if (!prompt) throw new Error('prompt is required');
|
|
395
|
+
|
|
396
|
+
let startResponse;
|
|
397
|
+
if (files && files.length > 0) {
|
|
398
|
+
// Multipart mode: resolve each file source to a buffer and upload.
|
|
399
|
+
const resolved = await Promise.all(files.map(src => resolveToBuffer(src, 'image')));
|
|
400
|
+
const form = new FormData();
|
|
401
|
+
form.append('prompt', prompt);
|
|
402
|
+
if (model) form.append('model', model);
|
|
403
|
+
if (duration !== undefined) form.append('duration', String(duration));
|
|
404
|
+
if (aspect_ratio) form.append('aspect_ratio', aspect_ratio);
|
|
405
|
+
if (motion) form.append('motion', motion);
|
|
406
|
+
if (preset_id) form.append('preset_id', preset_id);
|
|
407
|
+
if (enhance_prompt !== undefined) form.append('enhance_prompt', String(enhance_prompt));
|
|
408
|
+
if (visual_dna_ids) form.append('visual_dna_ids', JSON.stringify(visual_dna_ids));
|
|
409
|
+
if (reference_images) form.append('reference_images', JSON.stringify(reference_images));
|
|
410
|
+
for (const f of resolved) {
|
|
411
|
+
form.append('files', f.buffer, { filename: f.filename, contentType: f.contentType });
|
|
412
|
+
}
|
|
413
|
+
startResponse = await client.postMultipart('/v1/generate/elements', form);
|
|
414
|
+
} else {
|
|
415
|
+
// URL-only mode: plain JSON.
|
|
416
|
+
startResponse = await client.post('/v1/generate/elements', {
|
|
417
|
+
prompt, model, reference_images, duration, aspect_ratio, motion, preset_id, enhance_prompt, visual_dna_ids
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
const result = await pollUntilDone(client, startResponse.generation_id, {
|
|
422
|
+
interval: (startResponse.poll_interval_hint || 8) * 1000,
|
|
423
|
+
timeout: 600000
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
return {
|
|
427
|
+
content: [{
|
|
428
|
+
type: 'text',
|
|
429
|
+
text: JSON.stringify({
|
|
430
|
+
urls: result.result?.urls || [],
|
|
431
|
+
thumbnail_url: result.result?.thumbnail_url || null,
|
|
432
|
+
duration: result.result?.duration || null,
|
|
433
|
+
model: result.result?.model || null
|
|
434
|
+
}, null, 2)
|
|
435
|
+
}]
|
|
436
|
+
};
|
|
437
|
+
}
|
|
438
|
+
);
|
|
439
|
+
|
|
440
|
+
// ─── generate_first_last_frame ─────────────────────────────
|
|
441
|
+
server.tool(
|
|
442
|
+
'generate_first_last_frame',
|
|
443
|
+
'Generate a video that morphs / interpolates from a FIRST frame to a LAST frame. Provide the two frames as URLs (first_frame_url + last_frame_url) OR as local file paths (first_frame + last_frame). Optional prompt describes the desired motion/transition. Do NOT mix URL and file inputs. Returns the final video URL when complete.',
|
|
444
|
+
{
|
|
445
|
+
first_frame_url: z.string().optional().describe('Public URL of the first frame image (URL mode)'),
|
|
446
|
+
last_frame_url: z.string().optional().describe('Public URL of the last frame image (URL mode)'),
|
|
447
|
+
first_frame: z.string().optional().describe('URL or absolute local path to the first frame (file mode — alternative to first_frame_url)'),
|
|
448
|
+
last_frame: z.string().optional().describe('URL or absolute local path to the last frame (file mode — alternative to last_frame_url)'),
|
|
449
|
+
prompt: z.string().optional().describe('Optional description of the desired motion between the two frames (e.g. "smooth camera dolly in")'),
|
|
450
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="video_from_image" to see options. Omit for Smart Select.'),
|
|
451
|
+
duration: z.number().optional().describe('Duration in seconds. Default: 5'),
|
|
452
|
+
aspect_ratio: z.string().optional().describe('Aspect ratio (auto-detected from first frame if not provided). Default: "16:9"'),
|
|
453
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: true'),
|
|
454
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply.')
|
|
455
|
+
},
|
|
456
|
+
async ({ first_frame_url, last_frame_url, first_frame, last_frame, prompt, model, duration, aspect_ratio, enhance_prompt, visual_dna_ids }) => {
|
|
457
|
+
const urlMode = first_frame_url && last_frame_url;
|
|
458
|
+
const fileMode = first_frame && last_frame;
|
|
459
|
+
if (!urlMode && !fileMode) {
|
|
460
|
+
throw new Error('Provide either both first_frame_url + last_frame_url OR both first_frame + last_frame (URL/local path).');
|
|
461
|
+
}
|
|
462
|
+
if (urlMode && fileMode) {
|
|
463
|
+
throw new Error('Do not mix URL and file inputs. Provide either URLs OR file sources, not both.');
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
let startResponse;
|
|
467
|
+
if (fileMode) {
|
|
468
|
+
const [firstResolved, lastResolved] = await Promise.all([
|
|
469
|
+
resolveToBuffer(first_frame, 'image'),
|
|
470
|
+
resolveToBuffer(last_frame, 'image')
|
|
471
|
+
]);
|
|
472
|
+
const form = new FormData();
|
|
473
|
+
form.append('files', firstResolved.buffer, { filename: firstResolved.filename, contentType: firstResolved.contentType });
|
|
474
|
+
form.append('files', lastResolved.buffer, { filename: lastResolved.filename, contentType: lastResolved.contentType });
|
|
475
|
+
if (prompt) form.append('prompt', prompt);
|
|
476
|
+
if (model) form.append('model', model);
|
|
477
|
+
if (duration !== undefined) form.append('duration', String(duration));
|
|
478
|
+
if (aspect_ratio) form.append('aspect_ratio', aspect_ratio);
|
|
479
|
+
if (enhance_prompt !== undefined) form.append('enhance_prompt', String(enhance_prompt));
|
|
480
|
+
if (visual_dna_ids) form.append('visual_dna_ids', JSON.stringify(visual_dna_ids));
|
|
481
|
+
startResponse = await client.postMultipart('/v1/generate/first-last-frame', form);
|
|
482
|
+
} else {
|
|
483
|
+
startResponse = await client.post('/v1/generate/first-last-frame', {
|
|
484
|
+
first_frame_url, last_frame_url, prompt, model, duration, aspect_ratio, enhance_prompt, visual_dna_ids
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const result = await pollUntilDone(client, startResponse.generation_id, {
|
|
489
|
+
interval: (startResponse.poll_interval_hint || 8) * 1000,
|
|
490
|
+
timeout: 300000
|
|
491
|
+
});
|
|
492
|
+
|
|
493
|
+
return {
|
|
494
|
+
content: [{
|
|
495
|
+
type: 'text',
|
|
496
|
+
text: JSON.stringify({
|
|
497
|
+
urls: result.result?.urls || [],
|
|
498
|
+
thumbnail_url: result.result?.thumbnail_url || null,
|
|
499
|
+
duration: result.result?.duration || null,
|
|
500
|
+
model: result.result?.model || null
|
|
501
|
+
}, null, 2)
|
|
502
|
+
}]
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
);
|
|
506
|
+
|
|
507
|
+
// ─── generate_lipsync ──────────────────────────────────────
|
|
508
|
+
server.tool(
|
|
509
|
+
'generate_lipsync',
|
|
510
|
+
'Lipsync an audio track to a source image or video. Both `source` (image or video) and `audio` can be provided as URLs or as absolute local file paths. Pass a text_prompt only if the model supports it (some lipsync models do character performance from a prompt). Returns a lipsynced video URL.',
|
|
511
|
+
{
|
|
512
|
+
source: z.string().describe('URL or absolute local path to the source image or video (the face to animate)'),
|
|
513
|
+
audio: z.string().describe('URL or absolute local path to the audio track (the voice to sync to)'),
|
|
514
|
+
text_prompt: z.string().optional().describe('Optional text prompt (for performance-capable models)'),
|
|
515
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="lipsync" to see options. Omit for Smart Select.'),
|
|
516
|
+
bounding_box_target: z.array(z.number()).optional().describe('Optional bounding box [x, y, w, h] for multi-face inputs (Hedra Character3 style). Leave empty for single-face.')
|
|
517
|
+
},
|
|
518
|
+
async ({ source, audio, text_prompt, model, bounding_box_target }) => {
|
|
519
|
+
if (!source) throw new Error('source is required (URL or absolute local path to image/video)');
|
|
520
|
+
if (!audio) throw new Error('audio is required (URL or absolute local path to audio file)');
|
|
521
|
+
|
|
522
|
+
const sourceIsUrl = typeof source === 'string' && /^https?:\/\//i.test(source);
|
|
523
|
+
const audioIsUrl = typeof audio === 'string' && /^https?:\/\//i.test(audio);
|
|
524
|
+
|
|
525
|
+
let startResponse;
|
|
526
|
+
if (sourceIsUrl && audioIsUrl) {
|
|
527
|
+
// URL mode
|
|
528
|
+
startResponse = await client.post('/v1/generate/lipsync', {
|
|
529
|
+
source_url: source,
|
|
530
|
+
audio_url: audio,
|
|
531
|
+
prompt: text_prompt,
|
|
532
|
+
model,
|
|
533
|
+
bounding_box_target
|
|
534
|
+
});
|
|
535
|
+
} else {
|
|
536
|
+
// File mode (or mixed — resolve any local paths, pass URLs through as body fields)
|
|
537
|
+
const form = new FormData();
|
|
538
|
+
if (!sourceIsUrl) {
|
|
539
|
+
const resolved = await resolveToBuffer(source, /\.(mp4|mov|webm|mkv)$/i.test(source) ? 'video' : 'image');
|
|
540
|
+
// Decide field name by kind — lipsync controller uses .fields() with image/video/audio.
|
|
541
|
+
const isVideo = /\.(mp4|mov|webm|mkv|avi|m4v)$/i.test(resolved.filename);
|
|
542
|
+
form.append(isVideo ? 'video' : 'image', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
543
|
+
} else {
|
|
544
|
+
form.append('source_url', source);
|
|
545
|
+
}
|
|
546
|
+
if (!audioIsUrl) {
|
|
547
|
+
const resolved = await resolveToBuffer(audio, 'audio');
|
|
548
|
+
form.append('audio', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
549
|
+
} else {
|
|
550
|
+
form.append('audio_url', audio);
|
|
551
|
+
}
|
|
552
|
+
if (text_prompt) form.append('prompt', text_prompt);
|
|
553
|
+
if (model) form.append('model', model);
|
|
554
|
+
if (bounding_box_target) form.append('bounding_box_target', JSON.stringify(bounding_box_target));
|
|
555
|
+
startResponse = await client.postMultipart('/v1/generate/lipsync', form);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
const result = await pollUntilDone(client, startResponse.generation_id, {
|
|
559
|
+
interval: (startResponse.poll_interval_hint || 8) * 1000,
|
|
560
|
+
timeout: 600000
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
return {
|
|
564
|
+
content: [{
|
|
565
|
+
type: 'text',
|
|
566
|
+
text: JSON.stringify({
|
|
567
|
+
urls: result.result?.urls || [],
|
|
568
|
+
thumbnail_url: result.result?.thumbnail_url || null,
|
|
569
|
+
duration: result.result?.duration || null,
|
|
570
|
+
model: result.result?.model || null
|
|
571
|
+
}, null, 2)
|
|
572
|
+
}]
|
|
573
|
+
};
|
|
574
|
+
}
|
|
575
|
+
);
|
|
576
|
+
|
|
577
|
+
// ─── generate_video_from_video ─────────────────────────────
|
|
578
|
+
server.tool(
|
|
579
|
+
'generate_video_from_video',
|
|
580
|
+
'Restyle / transform an existing video using a text prompt (video-to-video). Use for style transfer, scene restyling, subject swap — anything where you want to keep the motion from the input video but change the look. Source video can be a URL or absolute local path. For animating a still image use generate_video_from_image instead. For text-only → video use generate_video.',
|
|
581
|
+
{
|
|
582
|
+
source_video: z.string().describe('URL or absolute local path to the source video to restyle'),
|
|
583
|
+
prompt: z.string().describe('Text description of the desired restyle / transformation'),
|
|
584
|
+
model: z.string().optional().describe('Model identifier. Omit for Smart Select.'),
|
|
585
|
+
aspect_ratio: z.string().optional().describe('Output aspect ratio. Default: matches source'),
|
|
586
|
+
duration: z.number().optional().describe('Duration in seconds (default: matches source)'),
|
|
587
|
+
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: true'),
|
|
588
|
+
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for character/style consistency.')
|
|
589
|
+
},
|
|
590
|
+
async ({ source_video, prompt, model, aspect_ratio, duration, enhance_prompt, visual_dna_ids }) => {
|
|
591
|
+
if (!source_video) throw new Error('source_video is required');
|
|
592
|
+
if (!prompt) throw new Error('prompt is required');
|
|
593
|
+
|
|
594
|
+
const isUrl = /^https?:\/\//i.test(source_video);
|
|
595
|
+
let startResponse;
|
|
596
|
+
if (isUrl) {
|
|
597
|
+
startResponse = await client.post('/v1/generate/video-from-video', {
|
|
598
|
+
video_url: source_video, prompt, model, aspect_ratio, duration, enhance_prompt, visual_dna_ids
|
|
599
|
+
});
|
|
600
|
+
} else {
|
|
601
|
+
const resolved = await resolveToBuffer(source_video, 'video');
|
|
602
|
+
const form = new FormData();
|
|
603
|
+
form.append('files', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
604
|
+
form.append('prompt', prompt);
|
|
605
|
+
if (model) form.append('model', model);
|
|
606
|
+
if (aspect_ratio) form.append('aspect_ratio', aspect_ratio);
|
|
607
|
+
if (duration !== undefined) form.append('duration', String(duration));
|
|
608
|
+
if (enhance_prompt !== undefined) form.append('enhance_prompt', String(enhance_prompt));
|
|
609
|
+
if (visual_dna_ids) form.append('visual_dna_ids', JSON.stringify(visual_dna_ids));
|
|
610
|
+
startResponse = await client.postMultipart('/v1/generate/video-from-video', form);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
const result = await pollUntilDone(client, startResponse.generation_id, {
|
|
614
|
+
interval: (startResponse.poll_interval_hint || 8) * 1000,
|
|
615
|
+
timeout: 600000
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
return {
|
|
619
|
+
content: [{
|
|
620
|
+
type: 'text',
|
|
621
|
+
text: JSON.stringify({
|
|
622
|
+
urls: result.result?.urls || [],
|
|
623
|
+
thumbnail_url: result.result?.thumbnail_url || null,
|
|
624
|
+
duration: result.result?.duration || null,
|
|
625
|
+
model: result.result?.model || null
|
|
626
|
+
}, null, 2)
|
|
627
|
+
}]
|
|
628
|
+
};
|
|
629
|
+
}
|
|
630
|
+
);
|
|
631
|
+
|
|
632
|
+
// ─── transcribe_audio ──────────────────────────────────────
|
|
633
|
+
server.tool(
|
|
634
|
+
'transcribe_audio',
|
|
635
|
+
'Transcribe audio or video into text + SRT subtitles. Source can be a URL or an absolute local file path. Returns the full text, SRT content, duration, and download URLs for .srt/.txt files. Works on both audio-only files (mp3, wav, m4a) and videos with audio tracks (mp4, mov, webm).',
|
|
636
|
+
{
|
|
637
|
+
source: z.string().describe('URL or absolute local path to the audio / video file to transcribe')
|
|
638
|
+
},
|
|
639
|
+
async ({ source }) => {
|
|
640
|
+
if (!source) throw new Error('source is required (URL or absolute local path)');
|
|
641
|
+
|
|
642
|
+
const isUrl = /^https?:\/\//i.test(source);
|
|
643
|
+
let startResponse;
|
|
644
|
+
if (isUrl) {
|
|
645
|
+
startResponse = await client.post('/v1/transcribe', { audio_url: source });
|
|
646
|
+
} else {
|
|
647
|
+
const resolved = await resolveToBuffer(source, 'audio');
|
|
648
|
+
const form = new FormData();
|
|
649
|
+
form.append('file', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
650
|
+
startResponse = await client.postMultipart('/v1/transcribe', form);
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const result = await pollUntilDone(client, startResponse.generation_id, {
|
|
654
|
+
interval: (startResponse.poll_interval_hint || 5) * 1000,
|
|
655
|
+
timeout: 1800000 // 30 minutes — long podcasts are a thing
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
return {
|
|
659
|
+
content: [{
|
|
660
|
+
type: 'text',
|
|
661
|
+
text: JSON.stringify({
|
|
662
|
+
text: result.result?.text || '',
|
|
663
|
+
srt_url: result.result?.srt_url || null,
|
|
664
|
+
txt_url: result.result?.txt_url || null,
|
|
665
|
+
duration: result.result?.duration || null
|
|
666
|
+
}, null, 2)
|
|
667
|
+
}]
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
);
|
|
671
|
+
|
|
672
|
+
// ─── generate_3d ───────────────────────────────────────────
|
|
673
|
+
server.tool(
|
|
674
|
+
'generate_3d',
|
|
675
|
+
'Generate a 3D model from a text prompt, a single reference image, or multiple reference images (for multi-view reconstruction). Returns model URLs in multiple formats (GLB, FBX, OBJ, USDZ). Modes: "text" (prompt-only), "single" (one image), "multi" (multiple images for better quality). The mode is auto-detected from the inputs if not specified.',
|
|
676
|
+
{
|
|
677
|
+
prompt: z.string().optional().describe('Text description of the 3D object to generate (used in text mode and also as a hint in image modes)'),
|
|
678
|
+
reference_images: z.array(z.string()).optional().describe('Array of public image URLs. 1 image → single mode, 2+ → multi mode.'),
|
|
679
|
+
mode: z.string().optional().describe('Explicitly set mode: "text" | "single" | "multi". Auto-detected from reference_images if omitted.'),
|
|
680
|
+
texture_prompt: z.string().optional().describe('Optional prompt to guide texture generation'),
|
|
681
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="three_d" to see options.'),
|
|
682
|
+
topology: z.string().optional().describe('Topology preset (optional, model-specific)'),
|
|
683
|
+
target_polycount: z.number().optional().describe('Target polygon count (optional, model-specific)'),
|
|
684
|
+
enable_tpose: z.boolean().optional().describe('Force T-pose for character models (optional)'),
|
|
685
|
+
enable_pbr: z.boolean().optional().describe('Enable PBR textures (optional)')
|
|
686
|
+
},
|
|
687
|
+
async ({ prompt, reference_images, mode, texture_prompt, model, topology, target_polycount, enable_tpose, enable_pbr }) => {
|
|
688
|
+
if (!prompt && !(reference_images && reference_images.length > 0)) {
|
|
689
|
+
throw new Error('Provide prompt (text mode) or reference_images (single/multi mode)');
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
const startResponse = await client.post('/v1/generate/3d', {
|
|
693
|
+
mode,
|
|
694
|
+
prompt,
|
|
695
|
+
reference_images,
|
|
696
|
+
texture_prompt,
|
|
697
|
+
model,
|
|
698
|
+
topology,
|
|
699
|
+
target_polycount,
|
|
700
|
+
enable_tpose,
|
|
701
|
+
enable_pbr
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
const result = await pollUntilDone(client, startResponse.generation_id, {
|
|
705
|
+
interval: (startResponse.poll_interval_hint || 8) * 1000,
|
|
706
|
+
timeout: 900000 // 15 minutes — 3D generation is slow
|
|
707
|
+
});
|
|
708
|
+
|
|
709
|
+
return {
|
|
710
|
+
content: [{
|
|
711
|
+
type: 'text',
|
|
712
|
+
text: JSON.stringify({
|
|
713
|
+
urls: result.result?.urls || [],
|
|
714
|
+
thumbnail_url: result.result?.thumbnail_url || null,
|
|
715
|
+
mode: result.result?.mode || null,
|
|
716
|
+
prompt_used: result.result?.prompt_used || null
|
|
717
|
+
}, null, 2)
|
|
718
|
+
}]
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
);
|
|
369
722
|
}
|
|
370
723
|
|
|
371
724
|
module.exports = { registerGenerateTools };
|