@kolbo/mcp 1.86.0 → 1.86.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/skill/SKILL.md +435 -434
- package/skill/references/models/3d.md +51 -0
- package/skill/references/models/seedance.md +32 -4
- package/skill/references/workflows/cost-and-validation.md +1 -1
- package/src/apps/index.js +750 -737
- package/src/apps/theme.js +13 -0
- package/src/apps/widgets/generation.js +16 -11
- package/src/apps/widgets/mediaGrid.js +14 -1
- package/src/apps/widgets/upload.js +456 -457
- package/src/index.js +6 -1
- package/src/toolAnnotations.js +1 -1
- package/src/tools/_shared.js +183 -32
- package/src/tools/artifacts.js +5 -2
- package/src/tools/generate.js +141 -36
- package/src/tools/media.js +656 -656
- package/src/tools/projects.js +15 -0
- package/src/tools/visual_dna.js +459 -456
- package/src/tools/voices.js +1 -1
package/src/tools/generate.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
const { z } = require('zod');
|
|
7
7
|
const FormData = require('form-data');
|
|
8
8
|
const { pollUntilDone, waitWindowMs } = require('../polling');
|
|
9
|
-
const { resolveToBuffer, pollOrTimedOut, creditFields, projectIdField, sessionIdField, inlineImageBlocks, linkFields, uiGenerating, uiCompleted, appsEnabled } = require('./_shared');
|
|
9
|
+
const { resolveToBuffer, pollOrTimedOut, creditFields, projectIdField, sessionIdField, inlineImageBlocks, linkFields, uiGenerating, asyncGenerating, uiCompleted, appsEnabled } = require('./_shared');
|
|
10
10
|
const { ownedUrl } = require('./owned-url');
|
|
11
11
|
const { UI, uiResult, canonicalModelId, assertModelSupportsType, modelInfo, voiceInfo, resolveCatalogAspectRatio } = require('../apps');
|
|
12
12
|
const { modelTypeForEditOperation, assertExecutableEditModel } = require('./editModelCatalog');
|
|
@@ -57,6 +57,7 @@ const CINEMATIC_SCHEMA = z.object({
|
|
|
57
57
|
// image is what varies, and that is the whole point). Either way the widget
|
|
58
58
|
// captions each tile with the item's prompt, so that is the label we carry.
|
|
59
59
|
const MAX_BATCH_PROMPTS = 8;
|
|
60
|
+
const MAX_STATUS_IDS = 20;
|
|
60
61
|
async function submitBatch(rawItems, submitOne) {
|
|
61
62
|
if (rawItems.length > MAX_BATCH_PROMPTS) {
|
|
62
63
|
throw new Error(
|
|
@@ -209,10 +210,30 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
209
210
|
// stdio hosts (Kolbo Code, Claude Desktop, Cursor) leave this false, so their
|
|
210
211
|
// tool output is unchanged: a text block with the image URL.
|
|
211
212
|
const inlineImages = !!options.inlineImages;
|
|
213
|
+
const resolveInput = (source, kind, opts = {}) => resolveToBuffer(source, kind, {
|
|
214
|
+
...opts,
|
|
215
|
+
allowLocalFiles: !options.remote,
|
|
216
|
+
});
|
|
217
|
+
// JSON-body tools (edit_image) cannot carry a file, so a local path is
|
|
218
|
+
// uploaded to the media library first and its CDN URL is sent instead.
|
|
219
|
+
// URLs pass through untouched.
|
|
220
|
+
const rehostLocal = async (source, kind, project_id) => {
|
|
221
|
+
if (!source || isUrlSource(source)) return source;
|
|
222
|
+
const file = await resolveInput(source, kind);
|
|
223
|
+
const form = new FormData();
|
|
224
|
+
form.append('file', file.buffer, { filename: file.filename, contentType: file.contentType });
|
|
225
|
+
if (project_id) form.append('project_id', project_id);
|
|
226
|
+
const uploaded = await client.postMultipart('/v1/media/upload', form);
|
|
227
|
+
const url = uploaded?.media?.url || uploaded?.url;
|
|
228
|
+
if (!url) throw new Error(`Upload of ${source} returned no URL`);
|
|
229
|
+
return url;
|
|
230
|
+
};
|
|
212
231
|
// MCP Apps hosts (claude.ai remote connector, Claude Desktop) get an instant
|
|
213
232
|
// "submitted" response + a live ui://kolbo/generation.html widget that keeps
|
|
214
233
|
// one wait=true status call in flight. Text-only hosts never take this branch.
|
|
215
234
|
const ui = () => appsEnabled(server, options);
|
|
235
|
+
const returnsImmediately = () => ui() || !!options.asyncGenerations;
|
|
236
|
+
const submittedResult = (params) => ui() ? uiGenerating(params) : asyncGenerating(params);
|
|
216
237
|
|
|
217
238
|
// How long the STATUS tools may block inside one tool call before handing
|
|
218
239
|
// back a non-terminal result the caller re-issues. Bounded by the transport,
|
|
@@ -260,7 +281,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
260
281
|
// Batch mode: N different prompts, one widget owning all generation ids.
|
|
261
282
|
if (prompts && prompts.length) {
|
|
262
283
|
const batch = await submitBatch(prompts, (p) => client.post('/v1/generate/image', { ...shared, prompt: p }));
|
|
263
|
-
if (
|
|
284
|
+
if (returnsImmediately()) return submittedResult({
|
|
264
285
|
tool: 'generate_image', kind: 'image', gen: batch.ok[0].gen, client, model,
|
|
265
286
|
count: batch.ids.length, settings: imageSettings(shared),
|
|
266
287
|
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
|
|
@@ -273,7 +294,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
273
294
|
|
|
274
295
|
const gen = await client.post('/v1/generate/image', { ...shared, prompt, num_images });
|
|
275
296
|
|
|
276
|
-
if (
|
|
297
|
+
if (returnsImmediately()) return submittedResult({
|
|
277
298
|
tool: 'generate_image', kind: 'image', gen, client, model, prompt,
|
|
278
299
|
count: num_images, settings: imageSettings(shared),
|
|
279
300
|
reference_images
|
|
@@ -349,7 +370,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
349
370
|
// back as four stacked cards.
|
|
350
371
|
if (prompts && prompts.length) {
|
|
351
372
|
const batch = await submitBatch(prompts, (p) => client.post('/v1/generate/image-edit', { ...shared, prompt: p }));
|
|
352
|
-
if (
|
|
373
|
+
if (returnsImmediately()) return submittedResult({
|
|
353
374
|
tool: 'generate_image_edit', kind: 'image', gen: batch.ok[0].gen, client, model,
|
|
354
375
|
count: batch.ids.length, settings,
|
|
355
376
|
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
|
|
@@ -362,7 +383,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
362
383
|
|
|
363
384
|
const gen = await client.post('/v1/generate/image-edit', { ...shared, prompt, num_images });
|
|
364
385
|
|
|
365
|
-
if (
|
|
386
|
+
if (returnsImmediately()) return submittedResult({
|
|
366
387
|
tool: 'generate_image_edit', kind: 'image', gen, client, model, prompt,
|
|
367
388
|
count: num_images,
|
|
368
389
|
settings,
|
|
@@ -429,7 +450,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
429
450
|
});
|
|
430
451
|
|
|
431
452
|
const cdStatusUrl = `/v1/generate/creative-director/${gen.generation_id}/status`;
|
|
432
|
-
if (
|
|
453
|
+
if (returnsImmediately()) return submittedResult({
|
|
433
454
|
tool: 'generate_creative_director', kind: 'scenes', gen, client, model, prompt,
|
|
434
455
|
count: scene_count || 4,
|
|
435
456
|
settings: videoSettings({ duration, resolution, aspect_ratio, mode: workflow_type || 'image', enhance_prompt, visual_dna_ids, moodboard_id, moodboard_ids }),
|
|
@@ -602,7 +623,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
602
623
|
// Batch mode: N different prompts, one widget owning all generation ids.
|
|
603
624
|
if (prompts && prompts.length) {
|
|
604
625
|
const batch = await submitBatch(prompts, (p) => client.post('/v1/generate/video', { ...shared, prompt: p }));
|
|
605
|
-
if (
|
|
626
|
+
if (returnsImmediately()) return submittedResult({
|
|
606
627
|
tool: 'generate_video', kind: 'video', gen: batch.ok[0].gen, client, model,
|
|
607
628
|
count: batch.ids.length, settings: videoSettings(shared),
|
|
608
629
|
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
|
|
@@ -615,7 +636,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
615
636
|
|
|
616
637
|
const gen = await client.post('/v1/generate/video', { ...shared, prompt });
|
|
617
638
|
|
|
618
|
-
if (
|
|
639
|
+
if (returnsImmediately()) return submittedResult({
|
|
619
640
|
tool: 'generate_video', kind: 'video', gen, client, model, prompt,
|
|
620
641
|
settings: videoSettings(shared),
|
|
621
642
|
reference_images
|
|
@@ -692,7 +713,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
692
713
|
// per-tile caption either way.
|
|
693
714
|
if (items && items.length) {
|
|
694
715
|
const batch = await submitBatch(items, (it) => client.post('/v1/generate/video/from-image', { ...shared, image_url: it.image_url, prompt: it.prompt }));
|
|
695
|
-
if (
|
|
716
|
+
if (returnsImmediately()) return submittedResult({
|
|
696
717
|
tool: 'generate_video_from_image', kind: 'video', gen: batch.ok[0].gen, client, model,
|
|
697
718
|
count: batch.ids.length, settings: videoSettings({ duration, resolution, aspect_ratio, enhance_prompt, visual_dna_ids }),
|
|
698
719
|
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
|
|
@@ -705,7 +726,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
705
726
|
|
|
706
727
|
const gen = await client.post('/v1/generate/video/from-image', { ...shared, image_url, prompt });
|
|
707
728
|
|
|
708
|
-
if (
|
|
729
|
+
if (returnsImmediately()) return submittedResult({
|
|
709
730
|
tool: 'generate_video_from_image', kind: 'video', gen, client, model, prompt,
|
|
710
731
|
settings: videoSettings({ duration, resolution, aspect_ratio, enhance_prompt, visual_dna_ids }),
|
|
711
732
|
reference_images: [image_url]
|
|
@@ -778,7 +799,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
778
799
|
singing_dna_id, singing_voice_id, project_id, session_id
|
|
779
800
|
});
|
|
780
801
|
|
|
781
|
-
if (
|
|
802
|
+
if (returnsImmediately()) return submittedResult({
|
|
782
803
|
tool: 'generate_music', kind: 'audio', gen, client, model: model || 'Suno', prompt,
|
|
783
804
|
settings: { mode: instrumental ? 'instrumental' : (style || undefined), preset_id },
|
|
784
805
|
});
|
|
@@ -877,7 +898,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
877
898
|
project_id, session_id
|
|
878
899
|
});
|
|
879
900
|
|
|
880
|
-
if (
|
|
901
|
+
if (returnsImmediately()) return submittedResult({
|
|
881
902
|
tool: 'generate_speech', kind: 'audio', gen, client, model, prompt: text,
|
|
882
903
|
voice: voiceRecord,
|
|
883
904
|
settings: {
|
|
@@ -963,7 +984,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
963
984
|
seed_reference_audio_urls, seed_reference_image_url, project_id, session_id
|
|
964
985
|
});
|
|
965
986
|
|
|
966
|
-
if (
|
|
987
|
+
if (returnsImmediately()) return submittedResult({
|
|
967
988
|
tool: 'generate_sound', kind: 'audio', gen, client, model, prompt,
|
|
968
989
|
settings: { duration },
|
|
969
990
|
reference_images: seed_reference_image_url ? [seed_reference_image_url] : []
|
|
@@ -1026,14 +1047,15 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1026
1047
|
`Check the status of one or more generations. Use after a generation tool returned "submitted" (widget hosts) or timed out. Tracking SEVERAL concurrent generations? Pass them ALL in generation_ids — one call returns an all_done summary. Need the final result? Set wait=true and the server blocks until every generation finishes, for at most ~${WAIT_WINDOW_S}s per call. A job that outlives one window (music is ~3 min, video longer) comes back state="processing" — that is a normal result, not an error: call again with wait=true and keep going until every id is terminal. Never poll with wait=false in a loop.`,
|
|
1027
1048
|
{
|
|
1028
1049
|
generation_id: z.string().optional().describe('A single generation ID to check'),
|
|
1029
|
-
generation_ids: z.array(z.string()).optional().describe(
|
|
1050
|
+
generation_ids: z.array(z.string()).max(MAX_STATUS_IDS).optional().describe(`Multiple generation IDs to check in ONE call (max ${MAX_STATUS_IDS}). Returns { all_done, pending, generations[] } — always prefer this over checking IDs one by one.`),
|
|
1030
1051
|
wait: z.boolean().optional().describe(`If true, block until every generation reaches a terminal state (completed/failed), for at most ~${WAIT_WINDOW_S}s per call, then return whatever state they are in. Anything still processing is reported, not errored — re-issue with wait=true and only the still-pending ids. This is always better than polling with wait=false.`)
|
|
1031
1052
|
},
|
|
1032
1053
|
async ({ generation_id, generation_ids, wait }) => {
|
|
1033
1054
|
const ids = (generation_ids && generation_ids.length > 0)
|
|
1034
|
-
? generation_ids
|
|
1055
|
+
? [...new Set(generation_ids)]
|
|
1035
1056
|
: (generation_id ? [generation_id] : []);
|
|
1036
1057
|
if (ids.length === 0) throw new Error('Provide generation_id or generation_ids');
|
|
1058
|
+
if (ids.length > MAX_STATUS_IDS) throw new Error(`At most ${MAX_STATUS_IDS} generation IDs can be checked at once.`);
|
|
1037
1059
|
|
|
1038
1060
|
// One status check (or blocking poll) per id. Never let one bad id
|
|
1039
1061
|
// reject the whole batch — surface it as a failed entry instead.
|
|
@@ -1064,6 +1086,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1064
1086
|
await Promise.all(results.map(addDisplayNames));
|
|
1065
1087
|
|
|
1066
1088
|
const pending = results.filter(r => r.state !== 'completed' && r.state !== 'failed' && r.state !== 'cancelled');
|
|
1089
|
+
const unresolved = results.filter(r => r.state === 'unknown');
|
|
1067
1090
|
const doneHint = 'ALL generations are in a final state — do NOT poll again. Report the results to the user.';
|
|
1068
1091
|
// The old wait=false hint said "call it ONCE with wait=true ... to block
|
|
1069
1092
|
// until they finish". That is the advice that broke: one wait=true call
|
|
@@ -1073,7 +1096,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1073
1096
|
const idsPhrase = pendingIds.length > 1
|
|
1074
1097
|
? ` and ONLY the still-pending ids: ${JSON.stringify(pendingIds)}`
|
|
1075
1098
|
: '';
|
|
1076
|
-
const pendingHint =
|
|
1099
|
+
const pendingHint = unresolved.length
|
|
1100
|
+
? `Status could not be verified for ${unresolved.map(r => r.generation_id).join(', ')}. Check the ID and connection, then retry once later; do not poll in a loop or submit a replacement generation.`
|
|
1101
|
+
: stillRunningHint(idsPhrase);
|
|
1077
1102
|
|
|
1078
1103
|
// Single-id calls keep the original flat TEXT shape — a live uiGenerating
|
|
1079
1104
|
// widget (ui()==true, still open from the original generate_* call) polls
|
|
@@ -1107,6 +1132,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1107
1132
|
// forever, so the flat fields have to live in structuredContent too.
|
|
1108
1133
|
const urls = preferOwned(Array.isArray(res.urls) ? res.urls : []);
|
|
1109
1134
|
const done = single.state === 'completed' && urls.length > 0;
|
|
1135
|
+
const extraContent = options.asyncGenerations && done && mediaKind(urls[0]) === 'image'
|
|
1136
|
+
? await inlineImageBlocks(urls, { enabled: inlineImages, maxCount: 1, maxBytes: 512 * 1024, fetchTimeoutMs: 3000 })
|
|
1137
|
+
: undefined;
|
|
1110
1138
|
return uiCompleted({
|
|
1111
1139
|
tool: 'get_generation_status', kind: done ? mediaKind(urls[0]) : 'status', client,
|
|
1112
1140
|
// The model that ACTUALLY ran. A single-id check resolves one
|
|
@@ -1135,6 +1163,14 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1135
1163
|
// the finished card shows every reference, including server-side
|
|
1136
1164
|
// additions (DNA stills, merged sources) the tool call never named.
|
|
1137
1165
|
reference_images: res.reference_images,
|
|
1166
|
+
// Transcription results ride the same status tool as media
|
|
1167
|
+
// generations; without these the transcript widget merges a payload
|
|
1168
|
+
// with no transcript in it.
|
|
1169
|
+
text: typeof res.text === 'string' ? res.text : undefined,
|
|
1170
|
+
srt_url: res.srt_url || undefined,
|
|
1171
|
+
word_by_word_srt_url: res.word_by_word_srt_url || undefined,
|
|
1172
|
+
txt_url: res.txt_url || undefined,
|
|
1173
|
+
audio_url: res.audio_url || undefined,
|
|
1138
1174
|
prompt: res.prompt_used || res.prompt || undefined,
|
|
1139
1175
|
credits_used: creditFields(single).credits_used,
|
|
1140
1176
|
items: [{
|
|
@@ -1143,7 +1179,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1143
1179
|
title: res.prompt_used || res.prompt || undefined,
|
|
1144
1180
|
url: Array.isArray(res.urls) ? res.urls[0] : undefined,
|
|
1145
1181
|
}],
|
|
1146
|
-
}, singleText);
|
|
1182
|
+
}, singleText, extraContent);
|
|
1147
1183
|
}
|
|
1148
1184
|
|
|
1149
1185
|
const text = JSON.stringify({
|
|
@@ -1185,6 +1221,13 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1185
1221
|
const actualRefs = results
|
|
1186
1222
|
.map(r => r.result && r.result.reference_images)
|
|
1187
1223
|
.find(refs => Array.isArray(refs) && refs.length);
|
|
1224
|
+
const completedImageUrls = results.flatMap((r) => {
|
|
1225
|
+
const urls = r?.result?.urls;
|
|
1226
|
+
return Array.isArray(urls) && urls.length && mediaKind(urls[0]) === 'image' ? urls : [];
|
|
1227
|
+
});
|
|
1228
|
+
const extraContent = options.asyncGenerations
|
|
1229
|
+
? await inlineImageBlocks(completedImageUrls, { enabled: inlineImages, maxCount: 1, maxBytes: 512 * 1024, fetchTimeoutMs: 3000 })
|
|
1230
|
+
: undefined;
|
|
1188
1231
|
return uiCompleted({
|
|
1189
1232
|
tool: 'get_generation_status', kind: 'status', client,
|
|
1190
1233
|
model: modelsRan.length === 1 ? modelsRan[0] : 'Generations',
|
|
@@ -1205,7 +1248,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1205
1248
|
url: Array.isArray(res.urls) ? res.urls[0] : undefined,
|
|
1206
1249
|
};
|
|
1207
1250
|
}),
|
|
1208
|
-
}, text);
|
|
1251
|
+
}, text, extraContent);
|
|
1209
1252
|
}
|
|
1210
1253
|
);
|
|
1211
1254
|
|
|
@@ -1345,7 +1388,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1345
1388
|
return client.post('/v1/generate/elements', body);
|
|
1346
1389
|
}
|
|
1347
1390
|
const resolved = await Promise.all(locals.map(({ src, kind }) =>
|
|
1348
|
-
|
|
1391
|
+
resolveInput(src, kind, { maxBytes: ELEMENTS_MAX_UPLOAD_BYTES })));
|
|
1349
1392
|
const form = new FormData();
|
|
1350
1393
|
for (const [key, value] of Object.entries(body)) {
|
|
1351
1394
|
if (value === undefined || value === null) continue;
|
|
@@ -1385,7 +1428,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1385
1428
|
for (const kind of ['image', 'video', 'audio']) {
|
|
1386
1429
|
media[kind] = await Promise.all(media[kind].map(async (src) => {
|
|
1387
1430
|
if (!isUrlSource(src)) return src;
|
|
1388
|
-
const file = await
|
|
1431
|
+
const file = await resolveInput(src, kind, { maxBytes: ELEMENTS_MAX_UPLOAD_BYTES });
|
|
1389
1432
|
const form = new FormData();
|
|
1390
1433
|
form.append('file', file.buffer, { filename: file.filename, contentType: file.contentType });
|
|
1391
1434
|
if (project_id) form.append('project_id', project_id);
|
|
@@ -1399,7 +1442,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1399
1442
|
startResponse = await startElements();
|
|
1400
1443
|
}
|
|
1401
1444
|
|
|
1402
|
-
if (
|
|
1445
|
+
if (returnsImmediately()) return submittedResult({
|
|
1403
1446
|
tool: 'generate_elements', kind: 'video', gen: startResponse, client, model, prompt,
|
|
1404
1447
|
settings: videoSettings({
|
|
1405
1448
|
duration,
|
|
@@ -1500,8 +1543,8 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1500
1543
|
let startResponse;
|
|
1501
1544
|
if (!isUrlSource(firstSource) || !isUrlSource(lastSource)) {
|
|
1502
1545
|
const [firstResolved, lastResolved] = await Promise.all([
|
|
1503
|
-
|
|
1504
|
-
|
|
1546
|
+
resolveInput(firstSource, 'image'),
|
|
1547
|
+
resolveInput(lastSource, 'image')
|
|
1505
1548
|
]);
|
|
1506
1549
|
const form = new FormData();
|
|
1507
1550
|
form.append('files', firstResolved.buffer, { filename: firstResolved.filename, contentType: firstResolved.contentType });
|
|
@@ -1524,7 +1567,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1524
1567
|
});
|
|
1525
1568
|
}
|
|
1526
1569
|
|
|
1527
|
-
if (
|
|
1570
|
+
if (returnsImmediately()) return submittedResult({
|
|
1528
1571
|
tool: 'generate_first_last_frame', kind: 'video', gen: startResponse, client, model, prompt,
|
|
1529
1572
|
settings: videoSettings({ duration, resolution, aspect_ratio, enhance_prompt, visual_dna_ids }),
|
|
1530
1573
|
reference_images: [firstSource, lastSource].filter(isUrlSource)
|
|
@@ -1614,7 +1657,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1614
1657
|
// File mode (or mixed — resolve any local paths, pass URLs through as body fields)
|
|
1615
1658
|
const form = new FormData();
|
|
1616
1659
|
if (!sourceIsUrl) {
|
|
1617
|
-
const resolved = await
|
|
1660
|
+
const resolved = await resolveInput(source, /\.(mp4|mov|webm|mkv)$/i.test(source) ? 'video' : 'image');
|
|
1618
1661
|
// Decide field name by kind — lipsync controller uses .fields() with image/video/audio.
|
|
1619
1662
|
const isVideo = /\.(mp4|mov|webm|mkv|avi|m4v)$/i.test(resolved.filename);
|
|
1620
1663
|
form.append(isVideo ? 'video' : 'image', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
@@ -1622,7 +1665,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1622
1665
|
form.append('source_url', source);
|
|
1623
1666
|
}
|
|
1624
1667
|
if (!audioIsUrl) {
|
|
1625
|
-
const resolved = await
|
|
1668
|
+
const resolved = await resolveInput(audio, 'audio');
|
|
1626
1669
|
form.append('audio', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
1627
1670
|
} else {
|
|
1628
1671
|
form.append('audio_url', audio);
|
|
@@ -1642,7 +1685,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1642
1685
|
startResponse = await client.postMultipart('/v1/generate/lipsync', form);
|
|
1643
1686
|
}
|
|
1644
1687
|
|
|
1645
|
-
if (
|
|
1688
|
+
if (returnsImmediately()) return submittedResult({
|
|
1646
1689
|
tool: 'generate_lipsync', kind: 'video', gen: startResponse, client, model,
|
|
1647
1690
|
prompt: text_prompt, settings: { mode: 'lipsync' },
|
|
1648
1691
|
reference_images: sourceIsUrl && !/\.(mp4|mov|webm|mkv|avi|m4v)(\?|$)/i.test(source) ? [source] : [],
|
|
@@ -1737,7 +1780,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1737
1780
|
project_id, session_id
|
|
1738
1781
|
});
|
|
1739
1782
|
} else {
|
|
1740
|
-
const resolved = await
|
|
1783
|
+
const resolved = await resolveInput(source_video, 'video');
|
|
1741
1784
|
const form = new FormData();
|
|
1742
1785
|
form.append('files', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
1743
1786
|
if (prompt) form.append('prompt', prompt);
|
|
@@ -1767,7 +1810,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1767
1810
|
startResponse = await client.postMultipart('/v1/generate/video-from-video', form);
|
|
1768
1811
|
}
|
|
1769
1812
|
|
|
1770
|
-
if (
|
|
1813
|
+
if (returnsImmediately()) return submittedResult({
|
|
1771
1814
|
tool: 'generate_video_from_video', kind: 'video', gen: startResponse, client, model,
|
|
1772
1815
|
prompt: prompt || (preset ? `Subtitles preset: ${preset}` : undefined),
|
|
1773
1816
|
settings: videoSettings({ duration, resolution, aspect_ratio, mode: preset ? 'subtitles' : 'restyle', enhance_prompt, visual_dna_ids }),
|
|
@@ -1842,7 +1885,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1842
1885
|
if (isUrl) {
|
|
1843
1886
|
startResponse = await client.post('/v1/transcribe', { audio_url: source, ...opts });
|
|
1844
1887
|
} else {
|
|
1845
|
-
const resolved = await
|
|
1888
|
+
const resolved = await resolveInput(source, 'audio');
|
|
1846
1889
|
const form = new FormData();
|
|
1847
1890
|
form.append('file', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
|
|
1848
1891
|
for (const [k, v] of Object.entries(opts)) {
|
|
@@ -1866,6 +1909,14 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1866
1909
|
});
|
|
1867
1910
|
}
|
|
1868
1911
|
|
|
1912
|
+
if (options.asyncGenerations) {
|
|
1913
|
+
return asyncGenerating({
|
|
1914
|
+
tool: 'transcribe_audio',
|
|
1915
|
+
kind: 'audio',
|
|
1916
|
+
gen: startResponse,
|
|
1917
|
+
});
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1869
1920
|
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
|
|
1870
1921
|
interval: (startResponse.poll_interval_hint || 5) * 1000,
|
|
1871
1922
|
timeout: 150000 // Return before host cutoff; LLM can call get_generation_status with wait=true for long podcasts
|
|
@@ -1893,7 +1944,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1893
1944
|
// ─── generate_3d ───────────────────────────────────────────
|
|
1894
1945
|
server.tool(
|
|
1895
1946
|
'generate_3d',
|
|
1896
|
-
'Generate a 3D model from a text prompt, a single reference image, or multiple reference images (for multi-view reconstruction). Returns model URLs in multiple formats (GLB, FBX, OBJ, USDZ). Modes: "text" (prompt-only), "single" (one image), "multi" (multiple images for better quality). The mode is auto-detected from the inputs if not specified.',
|
|
1947
|
+
'Generate a 3D model from a text prompt, a single reference image, or multiple reference images (for multi-view reconstruction). Returns model URLs in multiple formats (GLB, FBX, OBJ, USDZ). Modes: "text" (prompt-only), "single" (one image), "multi" (multiple images for better quality). The mode is auto-detected from the inputs if not specified. Three model families with family-scoped settings: Meshy V7 (mesh/texture controls should_remesh/should_texture/texture_image_url/symmetry_mode/enable_safety_checker, plus auto-rigging + animation: enable_rigging rigs a humanoid character with basic walk/run animation, enable_animation applies one of ~697 presets on top — requires enable_rigging; both cost extra credits), Trellis v1 (texture_size, guidance/sampling steps, mesh_simplify, multiimage_algo), Trellis 2 (resolution, t2_texture_size, decimation_target, remesh, tex_sampling_steps). Params for a different family than the selected model are ignored.',
|
|
1897
1948
|
{
|
|
1898
1949
|
prompt: z.string().optional().describe('Text description of the 3D object to generate (used in text mode and also as a hint in image modes)'),
|
|
1899
1950
|
reference_images: z.array(z.string()).optional().describe('Array of image URLs or absolute local paths. 1 image → single mode, 2+ → multi mode.'),
|
|
@@ -1903,10 +1954,34 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1903
1954
|
topology: z.string().optional().describe('Topology preset (optional, model-specific)'),
|
|
1904
1955
|
target_polycount: z.number().optional().describe('Target polygon count (optional, model-specific)'),
|
|
1905
1956
|
enable_tpose: z.boolean().optional().describe('Force T-pose for character models (optional)'),
|
|
1906
|
-
enable_pbr: z.boolean().optional().describe('Enable PBR textures (optional)'),
|
|
1957
|
+
enable_pbr: z.boolean().optional().describe('Enable PBR textures (optional). Requires should_texture.'),
|
|
1958
|
+
symmetry_mode: z.string().optional().describe('Symmetry control: "off" | "auto" | "on" (Meshy V7 only, default "auto")'),
|
|
1959
|
+
should_remesh: z.boolean().optional().describe('Rebuild the mesh with clean game-ready topology (Meshy V7 only, default true). False keeps the raw reconstructed mesh.'),
|
|
1960
|
+
should_texture: z.boolean().optional().describe('Generate textures (Meshy V7 only, default true). False produces an untextured mesh at a lower credit cost; disables enable_pbr/texture_prompt/texture_image_url.'),
|
|
1961
|
+
texture_image_url: z.string().optional().describe('URL of a 2D image to guide texturing (Meshy V7 only). Requires should_texture.'),
|
|
1962
|
+
enable_safety_checker: z.boolean().optional().describe('Screen input images before generation (Meshy V7 only, default true)'),
|
|
1963
|
+
enable_rigging: z.boolean().optional().describe('Auto-rig the model as a humanoid character with basic walk/run animations (Meshy V7 only, default false). Best results with clearly defined limbs. Costs extra credits.'),
|
|
1964
|
+
rigging_height_meters: z.number().optional().describe('Approximate character height in meters (Meshy V7 only, default 1.7). Only used when enable_rigging is true.'),
|
|
1965
|
+
enable_animation: z.boolean().optional().describe('Apply an animation preset to the rigged model (Meshy V7 only, default false). Requires enable_rigging. Costs extra credits on top of rigging.'),
|
|
1966
|
+
animation_action_id: z.number().optional().describe('Animation preset ID, 0-696, default 92 ("Idle") (Meshy V7 only). Only used when enable_animation is true. See docs.meshy.ai/en/api/animation-library for the preset catalog.'),
|
|
1967
|
+
art_style: z.string().optional().describe('"realistic" | "sculpture" (Meshy text mode only). Sculpture disables PBR.'),
|
|
1968
|
+
enable_prompt_expansion: z.boolean().optional().describe('AI-expand the prompt before generation (Meshy text mode only)'),
|
|
1969
|
+
seed: z.number().optional().describe('Seed for reproducibility (Trellis models only)'),
|
|
1970
|
+
texture_size: z.string().optional().describe('Texture resolution "512" | "1024" | "2048" (Trellis v1 only)'),
|
|
1971
|
+
resolution: z.string().optional().describe('Generation resolution "512" | "1024" | "1536" (Trellis 2 only)'),
|
|
1972
|
+
ss_guidance_strength: z.number().optional().describe('Sparse-structure guidance strength (Trellis only)'),
|
|
1973
|
+
ss_sampling_steps: z.number().optional().describe('Sparse-structure sampling steps (Trellis only) — more steps = higher quality, slower'),
|
|
1974
|
+
slat_guidance_strength: z.number().optional().describe('Structured-latent guidance strength (Trellis v1 only)'),
|
|
1975
|
+
slat_sampling_steps: z.number().optional().describe('Structured-latent sampling steps (Trellis only)'),
|
|
1976
|
+
mesh_simplify: z.number().optional().describe('Mesh simplification ratio (Trellis v1 only)'),
|
|
1977
|
+
multiimage_algo: z.string().optional().describe('"stochastic" | "multidiffusion" — multi-image fusion algorithm (Trellis v1 multi mode only)'),
|
|
1978
|
+
decimation_target: z.number().optional().describe('Target polygon count after decimation (Trellis 2 only)'),
|
|
1979
|
+
remesh: z.boolean().optional().describe('Remesh with projection (Trellis 2 only, default true)'),
|
|
1980
|
+
tex_sampling_steps: z.number().optional().describe('Texture sampling steps (Trellis 2 only)'),
|
|
1981
|
+
t2_texture_size: z.string().optional().describe('Texture resolution "1024" | "2048" | "4096" (Trellis 2 only)'),
|
|
1907
1982
|
project_id: projectIdField
|
|
1908
1983
|
},
|
|
1909
|
-
async ({ prompt, reference_images, mode, texture_prompt, model, topology, target_polycount, enable_tpose, enable_pbr, project_id }) => {
|
|
1984
|
+
async ({ prompt, reference_images, mode, texture_prompt, model, topology, target_polycount, enable_tpose, enable_pbr, symmetry_mode, should_remesh, should_texture, texture_image_url, enable_safety_checker, enable_rigging, rigging_height_meters, enable_animation, animation_action_id, art_style, enable_prompt_expansion, seed, texture_size, resolution, ss_guidance_strength, ss_sampling_steps, slat_guidance_strength, slat_sampling_steps, mesh_simplify, multiimage_algo, decimation_target, remesh, tex_sampling_steps, t2_texture_size, project_id }) => {
|
|
1910
1985
|
model = await canonicalModelId(client, model, ['3d_text_to_model', '3d_image_to_model', '3d_multi_image_to_model', '3d_world']); // lenient id resolution ("z-image" → "z-image/turbo")
|
|
1911
1986
|
if (!prompt && !(reference_images && reference_images.length > 0)) {
|
|
1912
1987
|
throw new Error('Provide prompt (text mode) or reference_images (single/multi mode)');
|
|
@@ -1922,10 +1997,34 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1922
1997
|
target_polycount,
|
|
1923
1998
|
enable_tpose,
|
|
1924
1999
|
enable_pbr,
|
|
2000
|
+
symmetry_mode,
|
|
2001
|
+
should_remesh,
|
|
2002
|
+
should_texture,
|
|
2003
|
+
texture_image_url,
|
|
2004
|
+
enable_safety_checker,
|
|
2005
|
+
enable_rigging,
|
|
2006
|
+
rigging_height_meters,
|
|
2007
|
+
enable_animation,
|
|
2008
|
+
animation_action_id,
|
|
2009
|
+
art_style,
|
|
2010
|
+
enable_prompt_expansion,
|
|
2011
|
+
seed,
|
|
2012
|
+
texture_size,
|
|
2013
|
+
resolution,
|
|
2014
|
+
ss_guidance_strength,
|
|
2015
|
+
ss_sampling_steps,
|
|
2016
|
+
slat_guidance_strength,
|
|
2017
|
+
slat_sampling_steps,
|
|
2018
|
+
mesh_simplify,
|
|
2019
|
+
multiimage_algo,
|
|
2020
|
+
decimation_target,
|
|
2021
|
+
remesh,
|
|
2022
|
+
tex_sampling_steps,
|
|
2023
|
+
t2_texture_size,
|
|
1925
2024
|
project_id
|
|
1926
2025
|
});
|
|
1927
2026
|
|
|
1928
|
-
if (
|
|
2027
|
+
if (returnsImmediately()) return submittedResult({
|
|
1929
2028
|
tool: 'generate_3d', kind: '3d', gen: startResponse, client, model, prompt,
|
|
1930
2029
|
settings: { mode: mode || (reference_images?.length > 1 ? 'multi' : reference_images?.length === 1 ? 'single' : 'text') },
|
|
1931
2030
|
reference_images
|
|
@@ -2082,6 +2181,12 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
2082
2181
|
throw new Error('mask_image_url (face reference) is required for face_swap');
|
|
2083
2182
|
}
|
|
2084
2183
|
|
|
2184
|
+
// Local paths → CDN URLs (image_url / mask_image_url / additional_images).
|
|
2185
|
+
[image_url, mask_image_url, additional_images] = await Promise.all([
|
|
2186
|
+
rehostLocal(image_url, 'image', project_id),
|
|
2187
|
+
rehostLocal(mask_image_url, 'image', project_id),
|
|
2188
|
+
additional_images ? Promise.all(additional_images.map((s) => rehostLocal(s, 'image', project_id))) : additional_images,
|
|
2189
|
+
]);
|
|
2085
2190
|
const gen = await client.post('/v1/edit/image', {
|
|
2086
2191
|
image_url, operation, model, scale, aspect_ratio, skin_strength, prompt,
|
|
2087
2192
|
mask_image_url, additional_images, generate_all_angles, resolution, quality, ai_optimize,
|
|
@@ -2090,7 +2195,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
2090
2195
|
project_id, session_id
|
|
2091
2196
|
});
|
|
2092
2197
|
|
|
2093
|
-
if (
|
|
2198
|
+
if (returnsImmediately()) return submittedResult({
|
|
2094
2199
|
tool: 'edit_image', kind: 'image', gen, client, model,
|
|
2095
2200
|
prompt: prompt || operation,
|
|
2096
2201
|
settings: { mode: operation, aspect_ratio, scale, resolution },
|
|
@@ -2274,7 +2379,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
2274
2379
|
project_id, session_id
|
|
2275
2380
|
});
|
|
2276
2381
|
|
|
2277
|
-
if (
|
|
2382
|
+
if (returnsImmediately()) return submittedResult({
|
|
2278
2383
|
tool: 'edit_video', kind: 'video', gen, client, model,
|
|
2279
2384
|
prompt: prompt || operation,
|
|
2280
2385
|
settings: { mode: operation, duration, aspect_ratio, resolution },
|