@sogni-ai/sogni-protocol 1.0.0-alpha.45 → 1.0.0-alpha.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/catalogs/audio-models.json +36 -36
- package/manifests/generation-tools.json +24 -5
- package/manifests/openai-tools.json +28 -9
- package/package.json +1 -1
- package/prompts/tools/sound_to_video.json +4 -2
- package/schemas/tools/generate_music.schema.json +8 -8
- package/schemas/tools/sound_to_video.schema.json +20 -1
- package/version.json +1 -1
|
@@ -1,41 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"description": "Audio / music generation model catalog. ACE-Step 1.5 family
|
|
3
|
-
"default": "
|
|
2
|
+
"description": "Audio / music generation model catalog. MiniMax Music 3 (the default) plus the ACE-Step 1.5 family. Each entry lists per-parameter ranges so SDKs can validate user input and render sliders without round-tripping to the server. Cross-cutting audio constraints (duration, BPM, time signatures) live in the top-level `constraints` block; models with different limits carry a per-model `constraints` override.",
|
|
3
|
+
"default": "music3",
|
|
4
4
|
"models": {
|
|
5
|
-
"turbo": {
|
|
6
|
-
"id": "ace_step_1.5_turbo",
|
|
7
|
-
"name": "ACE-Step 1.5 Turbo",
|
|
8
|
-
"steps": {
|
|
9
|
-
"min": 4,
|
|
10
|
-
"max": 16,
|
|
11
|
-
"default": 8
|
|
12
|
-
},
|
|
13
|
-
"shift": {
|
|
14
|
-
"min": 1,
|
|
15
|
-
"max": 5,
|
|
16
|
-
"default": 3
|
|
17
|
-
},
|
|
18
|
-
"guidance": null
|
|
19
|
-
},
|
|
20
|
-
"sft": {
|
|
21
|
-
"id": "ace_step_1.5_sft",
|
|
22
|
-
"name": "ACE-Step 1.5 SFT",
|
|
23
|
-
"steps": {
|
|
24
|
-
"min": 10,
|
|
25
|
-
"max": 200,
|
|
26
|
-
"default": 50
|
|
27
|
-
},
|
|
28
|
-
"shift": {
|
|
29
|
-
"min": 1,
|
|
30
|
-
"max": 5,
|
|
31
|
-
"default": 3
|
|
32
|
-
},
|
|
33
|
-
"guidance": {
|
|
34
|
-
"min": 1,
|
|
35
|
-
"max": 15,
|
|
36
|
-
"default": 5
|
|
37
|
-
}
|
|
38
|
-
},
|
|
39
5
|
"music3": {
|
|
40
6
|
"id": "minimax_music3",
|
|
41
7
|
"name": "MiniMax Music 3",
|
|
@@ -71,6 +37,40 @@
|
|
|
71
37
|
"bpm": null,
|
|
72
38
|
"timeSignatures": null
|
|
73
39
|
}
|
|
40
|
+
},
|
|
41
|
+
"turbo": {
|
|
42
|
+
"id": "ace_step_1.5_turbo",
|
|
43
|
+
"name": "ACE-Step 1.5 Turbo",
|
|
44
|
+
"steps": {
|
|
45
|
+
"min": 4,
|
|
46
|
+
"max": 16,
|
|
47
|
+
"default": 8
|
|
48
|
+
},
|
|
49
|
+
"shift": {
|
|
50
|
+
"min": 1,
|
|
51
|
+
"max": 5,
|
|
52
|
+
"default": 3
|
|
53
|
+
},
|
|
54
|
+
"guidance": null
|
|
55
|
+
},
|
|
56
|
+
"sft": {
|
|
57
|
+
"id": "ace_step_1.5_sft",
|
|
58
|
+
"name": "ACE-Step 1.5 SFT",
|
|
59
|
+
"steps": {
|
|
60
|
+
"min": 10,
|
|
61
|
+
"max": 200,
|
|
62
|
+
"default": 50
|
|
63
|
+
},
|
|
64
|
+
"shift": {
|
|
65
|
+
"min": 1,
|
|
66
|
+
"max": 5,
|
|
67
|
+
"default": 3
|
|
68
|
+
},
|
|
69
|
+
"guidance": {
|
|
70
|
+
"min": 1,
|
|
71
|
+
"max": 15,
|
|
72
|
+
"default": 5
|
|
73
|
+
}
|
|
74
74
|
}
|
|
75
75
|
},
|
|
76
76
|
"constraints": {
|
|
@@ -355,19 +355,19 @@
|
|
|
355
355
|
},
|
|
356
356
|
"duration": {
|
|
357
357
|
"type": "number",
|
|
358
|
-
"description": "Duration in seconds.
|
|
358
|
+
"description": "Duration in seconds. music3 (the default model): 10-300, default 60, and a ceiling — the song may end earlier at a musical resolution. ACE-Step turbo/sft: 10-600, default 30. Short clips: 10-30s. Standard songs: 120-300s.",
|
|
359
359
|
"minimum": 10,
|
|
360
360
|
"maximum": 600
|
|
361
361
|
},
|
|
362
362
|
"bpm": {
|
|
363
363
|
"type": "number",
|
|
364
|
-
"description": "Beats per minute / tempo. Default: 120. Range: 30-300. Slow ballad: 60-80. Mid-tempo: 90-120. Upbeat: 120-140. Fast dance: 140-180. Very fast: 180+.",
|
|
364
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the tempo in the prompt instead. Beats per minute / tempo. Default: 120. Range: 30-300. Slow ballad: 60-80. Mid-tempo: 90-120. Upbeat: 120-140. Fast dance: 140-180. Very fast: 180+.",
|
|
365
365
|
"minimum": 30,
|
|
366
366
|
"maximum": 300
|
|
367
367
|
},
|
|
368
368
|
"keyscale": {
|
|
369
369
|
"type": "string",
|
|
370
|
-
"description": "Musical key and scale. E.g., \"C major\", \"A minor\", \"F# minor\", \"Bb major\". Default: \"C major\". Only set when the user specifies a key or when a particular mood calls for it (minor keys for sad/dark, major for happy/bright)."
|
|
370
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the key in the prompt instead. Musical key and scale. E.g., \"C major\", \"A minor\", \"F# minor\", \"Bb major\". Default: \"C major\". Only set when the user specifies a key or when a particular mood calls for it (minor keys for sad/dark, major for happy/bright)."
|
|
371
371
|
},
|
|
372
372
|
"lyrics": {
|
|
373
373
|
"type": "string",
|
|
@@ -390,7 +390,7 @@
|
|
|
390
390
|
4,
|
|
391
391
|
6
|
|
392
392
|
],
|
|
393
|
-
"description": "Time signature (beats per measure). 4 = 4/4 time (default, most common). 3 = 3/4 time (waltz). 2 = 2/4 time (march). 6 = 6/8 time (compound). Default: 4."
|
|
393
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the time signature in the prompt instead. Time signature (beats per measure). 4 = 4/4 time (default, most common). 3 = 3/4 time (waltz). 2 = 2/4 time (march). 6 = 6/8 time (compound). Default: 4."
|
|
394
394
|
},
|
|
395
395
|
"numberOfVariations": {
|
|
396
396
|
"type": "number",
|
|
@@ -1405,7 +1405,7 @@
|
|
|
1405
1405
|
"minimax-h3-fasth3-a2v-turbo",
|
|
1406
1406
|
"minimax-h3-fasth3-a2v-turbo-2stage"
|
|
1407
1407
|
],
|
|
1408
|
-
"description": "\"ltx25-ia2v\" (default with image) and \"ltx25-a2v\" (default without image): LTX 2.5 image+audio and audio-only modes; Fast, HQ, and Pro currently use the release-validated official Distilled INT8 workflows. The Dev checkpoints are not publicly routed until upstream publishes and Sogni validates official ComfyUI Dev recipes. Video model. \"ltx23-ia2v\" (rollback with image): LTX 2.3 image+audio to video, audio-reactive with a reference image; Fast/HQ use the distilled 8-step worker and Default Media Quality Pro uses the non-distilled dev worker. \"ltx23-a2v\" (rollback without image): LTX 2.3 audio-only to video, no image needed, creates video purely from text prompt + audio with the same quality-tier routing. \"wan-s2v\": WAN 2.2 sound-to-video, best for lip-sync with a face image, fast 4-step. \"seedance2\": full Seedance 2.0 audio-reference video, 4-15s. \"seedance2-mini\": Seedance 2.0 Mini, 720p cap, fastest/lower-cost Seedance option. Seedance quality is selected only by this model value: pick \"seedance2-mini\" for faster/lower-cost drafts or explicit Mini requests, and pick \"seedance2\" for full-quality Seedance or 1080p/4K. Do not infer the Seedance model from Default Media Quality Fast/HQ/Pro. \"seedance2-5\": Seedance 2.5, the newest Seedance generation — 480p, 720p, and 1080p (4K is unsupported), 4-30s per clip at a fixed 24 fps, native audio, first-and-last-frame conditioning, and a much larger reference budget than the 2.0 family: up to 30 images, 10 videos, and 10 audios, with up to 50 reference media files total, subject to those per-modality caps. Choose \"seedance2-5\" when the user asks for Seedance 2.5, wants a single continuous Seedance clip longer than 15s (2.5 renders up to 30s in one call instead of being split and stitched), or wants a first-and-last-frame Seedance transition. Keep \"seedance2\" for 4K requests; Seedance 2.5 supports up to 1080p. For Seedance audio-reference prompts, preserve exact spoken dialogue when the user supplied it, and assign @Image1/@Audio1 roles. If the user asks for speech without words, describe the vocal performance without inventing quoted dialogue. Treat lip-sync, voice cloning, and real-human reference behavior as provider-sensitive rather than guaranteed. Omit to auto-select based on whether an image is present. \"wan3.0-video\" is Alibaba Wan 3 and \"wan3.0-spicy-video\" is MuleRouter w3.0-video. Both render 2-30s at fixed 30 fps with optional native audio, provider prompt expansion, 480p/720p/1080p, adaptive/fixed ratios, first/last frames, and up to 10 image/5 video/5 audio references. Only Alibaba wan3.0-video accepts document/web context and watermark. Frame anchors and loose references are mutually exclusive. Do not send negativePrompt; video references are loose conditioning for a new result, not source-video editing or extension. MiniMax H3 FastH3 audio guide: \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image from sourceImageIndex plus the audio), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame from sourceImageIndex, last frame from endImageIndex, plus the audio) and \"minimax-h3-fasth3-a2v-turbo\" (audio only; set neither sourceImageIndex nor endImageIndex) run the four-step FastH3 engine with the uploaded audio driving the picture from frame 0, and the output keeps that audio as its soundtrack. Choose them only when the user asks for MiniMax H3 or FastH3; never pick them in place of the LTX 2.5 defaults. They render 124-362 frames on the H3 17-frame grid at a fixed 24 fps (about 5.2-15.1 seconds) on a 32px grid within 1344x768, so use targetResolution 768 or omit it. audioStart picks the window of the upload. generateAudio=false
|
|
1408
|
+
"description": "\"ltx25-ia2v\" (default with image) and \"ltx25-a2v\" (default without image): LTX 2.5 image+audio and audio-only modes; Fast, HQ, and Pro currently use the release-validated official Distilled INT8 workflows. The Dev checkpoints are not publicly routed until upstream publishes and Sogni validates official ComfyUI Dev recipes. Video model. \"ltx23-ia2v\" (rollback with image): LTX 2.3 image+audio to video, audio-reactive with a reference image; Fast/HQ use the distilled 8-step worker and Default Media Quality Pro uses the non-distilled dev worker. \"ltx23-a2v\" (rollback without image): LTX 2.3 audio-only to video, no image needed, creates video purely from text prompt + audio with the same quality-tier routing. \"wan-s2v\": WAN 2.2 sound-to-video, best for lip-sync with a face image, fast 4-step. \"seedance2\": full Seedance 2.0 audio-reference video, 4-15s. \"seedance2-mini\": Seedance 2.0 Mini, 720p cap, fastest/lower-cost Seedance option. Seedance quality is selected only by this model value: pick \"seedance2-mini\" for faster/lower-cost drafts or explicit Mini requests, and pick \"seedance2\" for full-quality Seedance or 1080p/4K. Do not infer the Seedance model from Default Media Quality Fast/HQ/Pro. \"seedance2-5\": Seedance 2.5, the newest Seedance generation — 480p, 720p, and 1080p (4K is unsupported), 4-30s per clip at a fixed 24 fps, native audio, first-and-last-frame conditioning, and a much larger reference budget than the 2.0 family: up to 30 images, 10 videos, and 10 audios, with up to 50 reference media files total, subject to those per-modality caps. Choose \"seedance2-5\" when the user asks for Seedance 2.5, wants a single continuous Seedance clip longer than 15s (2.5 renders up to 30s in one call instead of being split and stitched), or wants a first-and-last-frame Seedance transition. Keep \"seedance2\" for 4K requests; Seedance 2.5 supports up to 1080p. For Seedance audio-reference prompts, preserve exact spoken dialogue when the user supplied it, and assign @Image1/@Audio1 roles. If the user asks for speech without words, describe the vocal performance without inventing quoted dialogue. Treat lip-sync, voice cloning, and real-human reference behavior as provider-sensitive rather than guaranteed. Omit to auto-select based on whether an image is present. \"wan3.0-video\" is Alibaba Wan 3 and \"wan3.0-spicy-video\" is MuleRouter w3.0-video. Both render 2-30s at fixed 30 fps with optional native audio, provider prompt expansion, 480p/720p/1080p, adaptive/fixed ratios, first/last frames, and up to 10 image/5 video/5 audio references. Only Alibaba wan3.0-video accepts document/web context and watermark. Frame anchors and loose references are mutually exclusive. Do not send negativePrompt; video references are loose conditioning for a new result, not source-video editing or extension. MiniMax H3 FastH3 audio guide: \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image from sourceImageIndex plus the audio), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame from sourceImageIndex, last frame from endImageIndex, plus the audio) and \"minimax-h3-fasth3-a2v-turbo\" (audio only; set neither sourceImageIndex nor endImageIndex) run the four-step FastH3 engine with the uploaded audio driving the picture from frame 0, and the output keeps that audio as its soundtrack. Choose them only when the user asks for MiniMax H3 or FastH3; never pick them in place of the LTX 2.5 defaults. They render 124-362 frames on the H3 17-frame grid at a fixed 24 fps (about 5.2-15.1 seconds) on a 32px grid within 1344x768, so use targetResolution 768 or omit it. audioStart picks the window of the upload. MiniMax H3 catalog and Personal LoRAs are supported across audio modes and tiers. generateAudio=false and negativePrompt are not supported. Each costs the FastH3 price of its image-to-video, first-and-last-frame or text-to-video mode, 4 Spark per second. \"minimax-h3-fasth3-ia2v-turbo-2stage\", \"minimax-h3-fasth3-flfa2v-turbo-2stage\" and \"minimax-h3-fasth3-a2v-turbo-2stage\" are the two-stage forms: the same inputs rendered on the FastH3 canvas, then enlarged 2x and refined, delivered at twice the canvas with the same length and audio. For them targetResolution names the delivered class: 1080 renders a 544px short-edge canvas (960x544 is delivered at 1920x1088) for 10 Spark per second, and 1440 or omitted renders the 768p canvas for 2K (1344x768 is delivered at 2688x1536) for 16 Spark per second. The audio two-stage selectors have no 720p price class (720 would render a 384px canvas at the 1080p rate), so for 720p or 768p audio-guided H3 output use the regular selector. The estimate prices every request."
|
|
1409
1409
|
},
|
|
1410
1410
|
"generateAudio": {
|
|
1411
1411
|
"type": "boolean",
|
|
@@ -1436,6 +1436,25 @@
|
|
|
1436
1436
|
"returnLastFrame": {
|
|
1437
1437
|
"type": "boolean",
|
|
1438
1438
|
"description": "Seedance 2.5 only. Set true to export a separate image of the final frame alongside the video. The result includes lastFrameUrl, which can be used as the first-frame image for a subsequent clip. Defaults to false; this does not extend the video automatically."
|
|
1439
|
+
},
|
|
1440
|
+
"loras": {
|
|
1441
|
+
"type": "array",
|
|
1442
|
+
"minItems": 1,
|
|
1443
|
+
"maxItems": 8,
|
|
1444
|
+
"items": {
|
|
1445
|
+
"type": "string",
|
|
1446
|
+
"minLength": 1
|
|
1447
|
+
},
|
|
1448
|
+
"description": "Ordered MiniMax H3 LoRA IDs, including owned ready Personal LoRAs. Stack up to 8 in one request; order matters because the adapters apply in sequence and do not commute. Keep this array positionally aligned with loraStrengths. The first render with an uncached LoRA takes longer to start while the worker downloads it.\n\nAccepted only when videoModel is one of \"minimax-h3-fasth3-ia2v-turbo\", \"minimax-h3-fasth3-ia2v-turbo-2stage\", \"minimax-h3-fasth3-flfa2v-turbo\", \"minimax-h3-fasth3-flfa2v-turbo-2stage\", \"minimax-h3-fasth3-a2v-turbo\", \"minimax-h3-fasth3-a2v-turbo-2stage\". Every other video model on this tool loads no LoRAs and silently ignores these arrays, so set videoModel to an H3 mode in the same call when the user asks for one.\n\nFive LoRAs are published for MiniMax H3 today and the set differs by mode, so GET /v1/loras/comfy?modelId=<model> is authoritative for the mode in hand and carries exact ranges, maturity flags, and anything published since. h3-realism-people (fal) is a realism pass trained on live-action footage of people: it restores skin texture and pores, stray hairs, fabric weave and a fine sensor grain that the base model smooths away, and holds up in close-up. It is the only one gated on a trigger word — put r34l1sm near the FRONT of the prompt, or the render comes back as ordinary H3 with no error. h3-vbvr-video-reasoning is a prompt-adherence pass that holds the model to what was asked instead of improvising. h3-natural-face-speech (AdaptiveVision) makes people talking on camera look and sound more natural: cheeks, brows, jaw and lips move together as in real speech, and spoken English comes through clearer; use it for talking-head shots such as vlogs, podcasts, interviews and presenters. h3-better-motion (AdaptiveVision) gives people more natural, consistent body movement — weight shifts, strides, turns and gestures that follow through — for dance, sport, walking and other full-body shots. Both AdaptiveVision LoRAs work best with short, simple prompt sentences and are not validated on reference-to-video. h3-mystic-xxx-v4 is an uncensored adult fine-tune. Personal imports are discovered through authenticated GET /v1/loras/personal/catalog; use only owned ready ids with the selected model in modelIds, and respect their strength range and requirements. Do not invent ids."
|
|
1449
|
+
},
|
|
1450
|
+
"loraStrengths": {
|
|
1451
|
+
"type": "array",
|
|
1452
|
+
"minItems": 1,
|
|
1453
|
+
"maxItems": 8,
|
|
1454
|
+
"items": {
|
|
1455
|
+
"type": "number"
|
|
1456
|
+
},
|
|
1457
|
+
"description": "Strength for each LoRA in loras, in the same order. Omitting the array applies 1.0 to every LoRA, which is not every LoRA's catalog default and for h3-realism-people is already at the top of its band, so send explicit values. Video LoRAs are positive-only — unlike the bipolar Krea 2 image sliders, a negative value is not an inverse effect and 0 is off. h3-realism-people takes 0-2 and its catalog default is 0.8; 0.6-1 is the usable band. It also pulls the camera in as it climbs: at 1.5 and above the shot reliably recomposes and the grade darkens, which on an image-conditioned mode can crop the subject out of the frame the user supplied. Raise it above 1 only when the user asks for more, and prefer the default when they supplied a first or last frame. h3-vbvr-video-reasoning and h3-mystic-xxx-v4 both take 0-1 and do default to 1.0, with usable bands of 0.7-1 and 0.2-1. h3-natural-face-speech and h3-better-motion take 0-1.5 and default to 0.6; their usable band is 0.4-0.8."
|
|
1439
1458
|
}
|
|
1440
1459
|
},
|
|
1441
1460
|
"required": [
|
|
@@ -326,27 +326,27 @@
|
|
|
326
326
|
"properties": {
|
|
327
327
|
"prompt": {
|
|
328
328
|
"type": "string",
|
|
329
|
-
"description": "Genre, mood, and style description for the music. Be specific about musical characteristics.\n\nLITERAL PROMPT OVERRIDE: If the user explicitly says not to modify the prompt, or to use it exactly/verbatim/as-is, copy the identified prompt text verbatim instead of applying these construction rules unless a hard requirement is missing.\n\nExamples:\n- \"upbeat electronic dance music with driving bass and synth arpeggios\"\n- \"mellow jazz ballad with soft piano, brushed drums, and walking bass\"\n- \"epic orchestral soundtrack with soaring strings and powerful brass\"\n- \"lo-fi hip hop beat with vinyl crackle, muted keys, and chill vibes\"\n- \"acoustic folk song with fingerpicked guitar and warm harmonies\"\n\nInclude:\n- Genre (rock, jazz, electronic, classical, hip-hop, etc.)\n- Mood (happy, melancholic, energetic, relaxing, epic, etc.)\n- Instruments (piano, guitar, drums, synth, strings, etc.)\n- Style descriptors (driving, mellow, atmospheric, punchy, etc.)\n\nBATCH VARIATIONS: When numberOfVariations > 1, use Dynamic Prompt syntax to vary ONE dimension across separate tracks. Lock in any genre/mood/instruments the user specified, vary the rest. Example: \"{lo-fi hip hop beat with muted keys|jazz piano trio with brushed drums|ambient electronic with soft pads} with warm reverb and vinyl texture\"."
|
|
329
|
+
"description": "Genre, mood, and style description for the music. Be specific about musical characteristics.\n\nLITERAL PROMPT OVERRIDE: If the user explicitly says not to modify the prompt, or to use it exactly/verbatim/as-is, copy the identified prompt text verbatim instead of applying these construction rules unless a hard requirement is missing.\n\nExamples:\n- \"upbeat electronic dance music with driving bass and synth arpeggios\"\n- \"mellow jazz ballad with soft piano, brushed drums, and walking bass\"\n- \"epic orchestral soundtrack with soaring strings and powerful brass\"\n- \"lo-fi hip hop beat with vinyl crackle, muted keys, and chill vibes\"\n- \"acoustic folk song with fingerpicked guitar and warm harmonies\"\n\nInclude:\n- Genre (rock, jazz, electronic, classical, hip-hop, etc.)\n- Mood (happy, melancholic, energetic, relaxing, epic, etc.)\n- Instruments (piano, guitar, drums, synth, strings, etc.)\n- Style descriptors (driving, mellow, atmospheric, punchy, etc.)\n\nMODEL \"music3\": MiniMax Music 3 wants a structured caption instead of a tag list — write the prompt as one paragraph in three labeled parts: \"Global Metadata: genre, BPM, key, emotional progression across the song, production profile. Vocal Details: gender, timbre, delivery, harmonies (or: none, purely instrumental). Arrangement: primary and secondary instruments, groove, bass, percussion, textures, how sections evolve.\" The more specific, the closer the result. Fold tempo and key into this caption — music3 ignores the bpm/keyscale/timesig args.\n\nBATCH VARIATIONS: When numberOfVariations > 1, use Dynamic Prompt syntax to vary ONE dimension across separate tracks. This is one Sogni project with multiple jobs, so prefer it when all tracks share the same duration, BPM, key, lyrics, model, and generation parameters and only prompt text varies. Lock in any genre/mood/instruments the user specified, vary the rest. Example: \"{lo-fi hip hop beat with muted keys|jazz piano trio with brushed drums|ambient electronic with soft pads} with warm reverb and vinyl texture\"."
|
|
330
330
|
},
|
|
331
331
|
"duration": {
|
|
332
332
|
"type": "number",
|
|
333
|
-
"description": "Duration in seconds.
|
|
333
|
+
"description": "Duration in seconds. music3 (the default model): 10-300, default 60, and a ceiling — the song may end earlier at a musical resolution. ACE-Step turbo/sft: 10-600, default 30. Short clips: 10-30s. Standard songs: 120-300s.",
|
|
334
334
|
"minimum": 10,
|
|
335
335
|
"maximum": 600
|
|
336
336
|
},
|
|
337
337
|
"bpm": {
|
|
338
338
|
"type": "number",
|
|
339
|
-
"description": "Beats per minute / tempo. Default: 120. Range: 30-300. Slow ballad: 60-80. Mid-tempo: 90-120. Upbeat: 120-140. Fast dance: 140-180. Very fast: 180+.",
|
|
339
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the tempo in the prompt instead. Beats per minute / tempo. Default: 120. Range: 30-300. Slow ballad: 60-80. Mid-tempo: 90-120. Upbeat: 120-140. Fast dance: 140-180. Very fast: 180+.",
|
|
340
340
|
"minimum": 30,
|
|
341
341
|
"maximum": 300
|
|
342
342
|
},
|
|
343
343
|
"keyscale": {
|
|
344
344
|
"type": "string",
|
|
345
|
-
"description": "Musical key and scale. E.g., \"C major\", \"A minor\", \"F# minor\", \"Bb major\". Default: \"C major\". Only set when the user specifies a key or when a particular mood calls for it (minor keys for sad/dark, major for happy/bright)."
|
|
345
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the key in the prompt instead. Musical key and scale. E.g., \"C major\", \"A minor\", \"F# minor\", \"Bb major\". Default: \"C major\". Only set when the user specifies a key or when a particular mood calls for it (minor keys for sad/dark, major for happy/bright)."
|
|
346
346
|
},
|
|
347
347
|
"lyrics": {
|
|
348
348
|
"type": "string",
|
|
349
|
-
"description": "Song lyrics. Optional — omit for instrumental music. Format: write lyrics naturally with line breaks. The model will attempt to sing these lyrics with the generated music. Works best with clear, rhythmic phrasing that matches the BPM."
|
|
349
|
+
"description": "Song lyrics. Optional — omit for instrumental music. Format: write lyrics naturally with line breaks. The model will attempt to sing these lyrics with the generated music. Works best with clear, rhythmic phrasing that matches the BPM. For model \"music3\", structure lyrics with plain section tags on their own lines ([Intro], [Verse], [Pre-Chorus], [Chorus], [Post-Chorus], [Bridge], [Solo], [Outro]) — no modifiers inside brackets, and write enough sections to fill the requested duration since the composer ends the song when the lyric sheet runs out. For instrumental music3 tracks, pass ONLY a skeleton of those tags one per line (e.g. [Intro] [Verse] [Chorus] [Verse] [Solo] [Chorus] [Outro]) — bare instrumental pieces end early without it."
|
|
350
350
|
},
|
|
351
351
|
"model": {
|
|
352
352
|
"type": "string",
|
|
@@ -355,7 +355,7 @@
|
|
|
355
355
|
"sft",
|
|
356
356
|
"music3"
|
|
357
357
|
],
|
|
358
|
-
"description": "Music model. \"
|
|
358
|
+
"description": "Music model. \"music3\" (default): MiniMax Music 3 — premium autoregressive composer with the best vocals, lyric adherence and song structure; 30 steps, up to 5 minutes, and it treats duration as a ceiling (may end the song early at a musical resolution). BPM/key/timesig args are ignored by music3 — fold tempo and key into the prompt instead. \"turbo\": ACE-Step 1.5 Turbo — fast 4-16 step drafts at roughly 1/20 the music3 cost; use only when the user asks for a quick, cheap, or draft track, or names ACE-Step. \"sft\": ACE-Step 1.5 SFT — experimental, strong lyric handling, 10-200 steps; use only when the user names it. Default to \"music3\" whenever the user does not ask for a draft or a specific model."
|
|
359
359
|
},
|
|
360
360
|
"timesig": {
|
|
361
361
|
"type": "number",
|
|
@@ -365,11 +365,11 @@
|
|
|
365
365
|
4,
|
|
366
366
|
6
|
|
367
367
|
],
|
|
368
|
-
"description": "Time signature (beats per measure). 4 = 4/4 time (default, most common). 3 = 3/4 time (waltz). 2 = 2/4 time (march). 6 = 6/8 time (compound). Default: 4."
|
|
368
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the time signature in the prompt instead. Time signature (beats per measure). 4 = 4/4 time (default, most common). 3 = 3/4 time (waltz). 2 = 2/4 time (march). 6 = 6/8 time (compound). Default: 4."
|
|
369
369
|
},
|
|
370
370
|
"numberOfVariations": {
|
|
371
371
|
"type": "number",
|
|
372
|
-
"description": "Number of variations (1-16). Use
|
|
372
|
+
"description": "Number of variations (1-16). Use with one Dynamic Prompt branch when the user requests multiple prompt-only music variations that share the same duration, BPM, key, lyrics, model, and parameters. This creates one Sogni project with multiple jobs. Default: 1.",
|
|
373
373
|
"minimum": 1,
|
|
374
374
|
"maximum": 16
|
|
375
375
|
}
|
|
@@ -1380,7 +1380,7 @@
|
|
|
1380
1380
|
"minimax-h3-fasth3-a2v-turbo",
|
|
1381
1381
|
"minimax-h3-fasth3-a2v-turbo-2stage"
|
|
1382
1382
|
],
|
|
1383
|
-
"description": "\"ltx25-ia2v\" (default with image) and \"ltx25-a2v\" (default without image): LTX 2.5 image+audio and audio-only modes; Fast, HQ, and Pro currently use the release-validated official Distilled INT8 workflows. The Dev checkpoints are not publicly routed until upstream publishes and Sogni validates official ComfyUI Dev recipes. Video model. \"ltx23-ia2v\" (rollback with image): LTX 2.3 image+audio to video, audio-reactive with a reference image; Fast/HQ use the distilled 8-step worker and Default Media Quality Pro uses the non-distilled dev worker. \"ltx23-a2v\" (rollback without image): LTX 2.3 audio-only to video, no image needed, creates video purely from text prompt + audio with the same quality-tier routing. \"wan-s2v\": WAN 2.2 sound-to-video, best for lip-sync with a face image, fast 4-step. \"seedance2\": full Seedance 2.0 audio-reference video, 4-15s. \"seedance2-mini\": Seedance 2.0 Mini, 720p cap, fastest/lower-cost Seedance option. Seedance quality is selected only by this model value: pick \"seedance2-mini\" for faster/lower-cost drafts or explicit Mini requests, and pick \"seedance2\" for full-quality Seedance or 1080p/4K. Do not infer the Seedance model from Default Media Quality Fast/HQ/Pro. \"seedance2-5\": Seedance 2.5, the newest Seedance generation — 480p, 720p, and 1080p (4K is unsupported), 4-30s per clip at a fixed 24 fps, native audio, first-and-last-frame conditioning, and a much larger reference budget than the 2.0 family: up to 30 images, 10 videos, and 10 audios, with up to 50 reference media files total, subject to those per-modality caps. Choose \"seedance2-5\" when the user asks for Seedance 2.5, wants a single continuous Seedance clip longer than 15s (2.5 renders up to 30s in one call instead of being split and stitched), or wants a first-and-last-frame Seedance transition. Keep \"seedance2\" for 4K requests; Seedance 2.5 supports up to 1080p. For Seedance audio-reference prompts, preserve exact spoken dialogue when the user supplied it, and assign @Image1/@Audio1 roles. If the user asks for speech without words, describe the vocal performance without inventing quoted dialogue. Treat lip-sync, voice cloning, and real-human reference behavior as provider-sensitive rather than guaranteed. Omit to auto-select based on whether an image is present. \"wan3.0-video\" is Alibaba Wan 3 and \"wan3.0-spicy-video\" is MuleRouter w3.0-video. Both render 2-30s at fixed 30 fps with optional native audio, provider prompt expansion, 480p/720p/1080p, adaptive/fixed ratios, first/last frames, and up to 10 image/5 video/5 audio references. Only Alibaba wan3.0-video accepts document/web context and watermark. Frame anchors and loose references are mutually exclusive. Do not send negativePrompt; video references are loose conditioning for a new result, not source-video editing or extension. MiniMax H3 FastH3 audio guide: \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image from sourceImageIndex plus the audio), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame from sourceImageIndex, last frame from endImageIndex, plus the audio) and \"minimax-h3-fasth3-a2v-turbo\" (audio only; set neither sourceImageIndex nor endImageIndex) run the four-step FastH3 engine with the uploaded audio driving the picture from frame 0, and the output keeps that audio as its soundtrack. Choose them only when the user asks for MiniMax H3 or FastH3; never pick them in place of the LTX 2.5 defaults. They render 124-362 frames on the H3 17-frame grid at a fixed 24 fps (about 5.2-15.1 seconds) on a 32px grid within 1344x768, so use targetResolution 768 or omit it. audioStart picks the window of the upload. generateAudio=false
|
|
1383
|
+
"description": "\"ltx25-ia2v\" (default with image) and \"ltx25-a2v\" (default without image): LTX 2.5 image+audio and audio-only modes; Fast, HQ, and Pro currently use the release-validated official Distilled INT8 workflows. The Dev checkpoints are not publicly routed until upstream publishes and Sogni validates official ComfyUI Dev recipes. Video model. \"ltx23-ia2v\" (rollback with image): LTX 2.3 image+audio to video, audio-reactive with a reference image; Fast/HQ use the distilled 8-step worker and Default Media Quality Pro uses the non-distilled dev worker. \"ltx23-a2v\" (rollback without image): LTX 2.3 audio-only to video, no image needed, creates video purely from text prompt + audio with the same quality-tier routing. \"wan-s2v\": WAN 2.2 sound-to-video, best for lip-sync with a face image, fast 4-step. \"seedance2\": full Seedance 2.0 audio-reference video, 4-15s. \"seedance2-mini\": Seedance 2.0 Mini, 720p cap, fastest/lower-cost Seedance option. Seedance quality is selected only by this model value: pick \"seedance2-mini\" for faster/lower-cost drafts or explicit Mini requests, and pick \"seedance2\" for full-quality Seedance or 1080p/4K. Do not infer the Seedance model from Default Media Quality Fast/HQ/Pro. \"seedance2-5\": Seedance 2.5, the newest Seedance generation — 480p, 720p, and 1080p (4K is unsupported), 4-30s per clip at a fixed 24 fps, native audio, first-and-last-frame conditioning, and a much larger reference budget than the 2.0 family: up to 30 images, 10 videos, and 10 audios, with up to 50 reference media files total, subject to those per-modality caps. Choose \"seedance2-5\" when the user asks for Seedance 2.5, wants a single continuous Seedance clip longer than 15s (2.5 renders up to 30s in one call instead of being split and stitched), or wants a first-and-last-frame Seedance transition. Keep \"seedance2\" for 4K requests; Seedance 2.5 supports up to 1080p. For Seedance audio-reference prompts, preserve exact spoken dialogue when the user supplied it, and assign @Image1/@Audio1 roles. If the user asks for speech without words, describe the vocal performance without inventing quoted dialogue. Treat lip-sync, voice cloning, and real-human reference behavior as provider-sensitive rather than guaranteed. Omit to auto-select based on whether an image is present. \"wan3.0-video\" is Alibaba Wan 3 and \"wan3.0-spicy-video\" is MuleRouter w3.0-video. Both render 2-30s at fixed 30 fps with optional native audio, provider prompt expansion, 480p/720p/1080p, adaptive/fixed ratios, first/last frames, and up to 10 image/5 video/5 audio references. Only Alibaba wan3.0-video accepts document/web context and watermark. Frame anchors and loose references are mutually exclusive. Do not send negativePrompt; video references are loose conditioning for a new result, not source-video editing or extension. MiniMax H3 FastH3 audio guide: \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image from sourceImageIndex plus the audio), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame from sourceImageIndex, last frame from endImageIndex, plus the audio) and \"minimax-h3-fasth3-a2v-turbo\" (audio only; set neither sourceImageIndex nor endImageIndex) run the four-step FastH3 engine with the uploaded audio driving the picture from frame 0, and the output keeps that audio as its soundtrack. Choose them only when the user asks for MiniMax H3 or FastH3; never pick them in place of the LTX 2.5 defaults. They render 124-362 frames on the H3 17-frame grid at a fixed 24 fps (about 5.2-15.1 seconds) on a 32px grid within 1344x768, so use targetResolution 768 or omit it. audioStart picks the window of the upload. MiniMax H3 catalog and Personal LoRAs are supported across audio modes and tiers. generateAudio=false and negativePrompt are not supported. Each costs the FastH3 price of its image-to-video, first-and-last-frame or text-to-video mode, 4 Spark per second. \"minimax-h3-fasth3-ia2v-turbo-2stage\", \"minimax-h3-fasth3-flfa2v-turbo-2stage\" and \"minimax-h3-fasth3-a2v-turbo-2stage\" are the two-stage forms: the same inputs rendered on the FastH3 canvas, then enlarged 2x and refined, delivered at twice the canvas with the same length and audio. For them targetResolution names the delivered class: 1080 renders a 544px short-edge canvas (960x544 is delivered at 1920x1088) for 10 Spark per second, and 1440 or omitted renders the 768p canvas for 2K (1344x768 is delivered at 2688x1536) for 16 Spark per second. The audio two-stage selectors have no 720p price class (720 would render a 384px canvas at the 1080p rate), so for 720p or 768p audio-guided H3 output use the regular selector. The estimate prices every request."
|
|
1384
1384
|
},
|
|
1385
1385
|
"generateAudio": {
|
|
1386
1386
|
"type": "boolean",
|
|
@@ -1411,6 +1411,25 @@
|
|
|
1411
1411
|
"returnLastFrame": {
|
|
1412
1412
|
"type": "boolean",
|
|
1413
1413
|
"description": "Seedance 2.5 only. Set true to export a separate image of the final frame alongside the video. The result includes lastFrameUrl, which can be used as the first-frame image for a subsequent clip. Defaults to false; this does not extend the video automatically."
|
|
1414
|
+
},
|
|
1415
|
+
"loras": {
|
|
1416
|
+
"type": "array",
|
|
1417
|
+
"minItems": 1,
|
|
1418
|
+
"maxItems": 8,
|
|
1419
|
+
"items": {
|
|
1420
|
+
"type": "string",
|
|
1421
|
+
"minLength": 1
|
|
1422
|
+
},
|
|
1423
|
+
"description": "Ordered MiniMax H3 LoRA IDs, including owned ready Personal LoRAs. Stack up to 8 in one request; order matters because the adapters apply in sequence and do not commute. Keep this array positionally aligned with loraStrengths. The first render with an uncached LoRA takes longer to start while the worker downloads it.\n\nAccepted only when videoModel is one of \"minimax-h3-fasth3-ia2v-turbo\", \"minimax-h3-fasth3-ia2v-turbo-2stage\", \"minimax-h3-fasth3-flfa2v-turbo\", \"minimax-h3-fasth3-flfa2v-turbo-2stage\", \"minimax-h3-fasth3-a2v-turbo\", \"minimax-h3-fasth3-a2v-turbo-2stage\". Every other video model on this tool loads no LoRAs and silently ignores these arrays, so set videoModel to an H3 mode in the same call when the user asks for one.\n\nFive LoRAs are published for MiniMax H3 today and the set differs by mode, so GET /v1/loras/comfy?modelId=<model> is authoritative for the mode in hand and carries exact ranges, maturity flags, and anything published since. h3-realism-people (fal) is a realism pass trained on live-action footage of people: it restores skin texture and pores, stray hairs, fabric weave and a fine sensor grain that the base model smooths away, and holds up in close-up. It is the only one gated on a trigger word — put r34l1sm near the FRONT of the prompt, or the render comes back as ordinary H3 with no error. h3-vbvr-video-reasoning is a prompt-adherence pass that holds the model to what was asked instead of improvising. h3-natural-face-speech (AdaptiveVision) makes people talking on camera look and sound more natural: cheeks, brows, jaw and lips move together as in real speech, and spoken English comes through clearer; use it for talking-head shots such as vlogs, podcasts, interviews and presenters. h3-better-motion (AdaptiveVision) gives people more natural, consistent body movement — weight shifts, strides, turns and gestures that follow through — for dance, sport, walking and other full-body shots. Both AdaptiveVision LoRAs work best with short, simple prompt sentences and are not validated on reference-to-video. h3-mystic-xxx-v4 is an uncensored adult fine-tune. Personal imports are discovered through authenticated GET /v1/loras/personal/catalog; use only owned ready ids with the selected model in modelIds, and respect their strength range and requirements. Do not invent ids."
|
|
1424
|
+
},
|
|
1425
|
+
"loraStrengths": {
|
|
1426
|
+
"type": "array",
|
|
1427
|
+
"minItems": 1,
|
|
1428
|
+
"maxItems": 8,
|
|
1429
|
+
"items": {
|
|
1430
|
+
"type": "number"
|
|
1431
|
+
},
|
|
1432
|
+
"description": "Strength for each LoRA in loras, in the same order. Omitting the array applies 1.0 to every LoRA, which is not every LoRA's catalog default and for h3-realism-people is already at the top of its band, so send explicit values. Video LoRAs are positive-only — unlike the bipolar Krea 2 image sliders, a negative value is not an inverse effect and 0 is off. h3-realism-people takes 0-2 and its catalog default is 0.8; 0.6-1 is the usable band. It also pulls the camera in as it climbs: at 1.5 and above the shot reliably recomposes and the grade darkens, which on an image-conditioned mode can crop the subject out of the frame the user supplied. Raise it above 1 only when the user asks for more, and prefer the default when they supplied a first or last frame. h3-vbvr-video-reasoning and h3-mystic-xxx-v4 both take 0-1 and do default to 1.0, with usable bands of 0.7-1 and 0.2-1. h3-natural-face-speech and h3-better-motion take 0-1.5 and default to 0.6; their usable band is 0.4-0.8."
|
|
1414
1433
|
}
|
|
1415
1434
|
},
|
|
1416
1435
|
"required": [
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sogni-ai/sogni-protocol",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.47",
|
|
4
4
|
"description": "Language-neutral protocol artifacts for the Sogni ecosystem: tool schemas, prompts, OpenAI tool manifests, and enums. Consumed by every Sogni SDK (TypeScript, Swift, and future Python/Kotlin/Rust SDKs) so contracts stay in lockstep across languages.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"sogni",
|
|
@@ -2,10 +2,12 @@
|
|
|
2
2
|
"contractId": "sound_to_video_v1",
|
|
3
3
|
"version": "1.1.0",
|
|
4
4
|
"toolName": "sound_to_video",
|
|
5
|
-
"baseDescription": "sound_to_video creates audio-synced video from an audio source. Works with uploaded audio\nfiles (mp3, m4a, wav) OR previously generated music from generate_music (auto-detected).\n\nWhen the user asks to \"turn that song/music into a video\" after generate_music, use\nsound_to_video — it will automatically find the generated audio.\n\nFor music visualization (syncing video to a specific song or audio track), use the\ngenerate_music → sound_to_video pipeline. Do NOT use animate_photo or generate_video for\naudio-driven visualization.\n\nanimate_photo and generate_video produce audio natively via LTX 2.5 by default, with LTX 2.3 retained as rollback — never pre-generate\naudio for those tools. sound_to_video is only for when the audio IS the primary creative\ninput driving the video output.\n\nMINIMAX H3 AUDIO GUIDE: only when the user asks for MiniMax H3 or FastH3 with an uploaded audio\ntrack, use sound_to_video with videoModel \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image via\nsourceImageIndex), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame via sourceImageIndex and last\nframe via endImageIndex) or \"minimax-h3-fasth3-a2v-turbo\" (audio only, no image). The audio\ndrives the picture from frame 0 and stays in the output. Clips are 124-362 frames at a fixed\n24 fps (about 5.2-15.1 seconds); audioStart picks the window. generateAudio=false
|
|
5
|
+
"baseDescription": "sound_to_video creates audio-synced video from an audio source. Works with uploaded audio\nfiles (mp3, m4a, wav) OR previously generated music from generate_music (auto-detected).\n\nWhen the user asks to \"turn that song/music into a video\" after generate_music, use\nsound_to_video — it will automatically find the generated audio.\n\nFor music visualization (syncing video to a specific song or audio track), use the\ngenerate_music → sound_to_video pipeline. Do NOT use animate_photo or generate_video for\naudio-driven visualization.\n\nanimate_photo and generate_video produce audio natively via LTX 2.5 by default, with LTX 2.3 retained as rollback — never pre-generate\naudio for those tools. sound_to_video is only for when the audio IS the primary creative\ninput driving the video output.\n\nMINIMAX H3 AUDIO GUIDE: only when the user asks for MiniMax H3 or FastH3 with an uploaded audio\ntrack, use sound_to_video with videoModel \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image via\nsourceImageIndex), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame via sourceImageIndex and last\nframe via endImageIndex) or \"minimax-h3-fasth3-a2v-turbo\" (audio only, no image). The audio\ndrives the picture from frame 0 and stays in the output. Clips are 124-362 frames at a fixed\n24 fps (about 5.2-15.1 seconds); audioStart picks the window. MiniMax H3 catalog and Personal LoRAs are supported across audio modes and tiers. generateAudio=false and\nnegativePrompt are not supported. Add \"-2stage\" only for 1080p, 1440p or 2K H3 output (1080 or\n1440 targetResolution); 720p-class audio output stays on the regular selector. Never use these\nin place of the LTX 2.5 defaults when the user did not ask for MiniMax H3 or FastH3.",
|
|
6
6
|
"parameterDocs": {
|
|
7
7
|
"audioSource": "Uploaded audio file or reference to a prior generate_music result. Auto-detected when omitted after generate_music.",
|
|
8
8
|
"negativePrompt": "Advanced LTX 2.5/LTX 2.3/WAN only. The LTX A2V and IA2V workflows accept a separate negative prompt; do not set it for Seedance.",
|
|
9
|
-
"endImageIndex": "Last frame for the MiniMax H3 FastH3 first/last frame + audio selectors only, indexed like animate_photo (-1 first upload, -2 second upload, non-negative generated results)."
|
|
9
|
+
"endImageIndex": "Last frame for the MiniMax H3 FastH3 first/last frame + audio selectors only, indexed like animate_photo (-1 first upload, -2 second upload, non-negative generated results).",
|
|
10
|
+
"loras": "Ordered MiniMax H3 catalog or owned ready Personal LoRA IDs, across audio workflows and tiers.",
|
|
11
|
+
"loraStrengths": "One strength per LoRA, in the same order; use the range from its catalog entry."
|
|
10
12
|
}
|
|
11
13
|
}
|
|
@@ -9,27 +9,27 @@
|
|
|
9
9
|
"properties": {
|
|
10
10
|
"prompt": {
|
|
11
11
|
"type": "string",
|
|
12
|
-
"description": "Genre, mood, and style description for the music. Be specific about musical characteristics.\n\nLITERAL PROMPT OVERRIDE: If the user explicitly says not to modify the prompt, or to use it exactly/verbatim/as-is, copy the identified prompt text verbatim instead of applying these construction rules unless a hard requirement is missing.\n\nExamples:\n- \"upbeat electronic dance music with driving bass and synth arpeggios\"\n- \"mellow jazz ballad with soft piano, brushed drums, and walking bass\"\n- \"epic orchestral soundtrack with soaring strings and powerful brass\"\n- \"lo-fi hip hop beat with vinyl crackle, muted keys, and chill vibes\"\n- \"acoustic folk song with fingerpicked guitar and warm harmonies\"\n\nInclude:\n- Genre (rock, jazz, electronic, classical, hip-hop, etc.)\n- Mood (happy, melancholic, energetic, relaxing, epic, etc.)\n- Instruments (piano, guitar, drums, synth, strings, etc.)\n- Style descriptors (driving, mellow, atmospheric, punchy, etc.)\n\nBATCH VARIATIONS: When numberOfVariations > 1, use Dynamic Prompt syntax to vary ONE dimension across separate tracks. Lock in any genre/mood/instruments the user specified, vary the rest. Example: \"{lo-fi hip hop beat with muted keys|jazz piano trio with brushed drums|ambient electronic with soft pads} with warm reverb and vinyl texture\"."
|
|
12
|
+
"description": "Genre, mood, and style description for the music. Be specific about musical characteristics.\n\nLITERAL PROMPT OVERRIDE: If the user explicitly says not to modify the prompt, or to use it exactly/verbatim/as-is, copy the identified prompt text verbatim instead of applying these construction rules unless a hard requirement is missing.\n\nExamples:\n- \"upbeat electronic dance music with driving bass and synth arpeggios\"\n- \"mellow jazz ballad with soft piano, brushed drums, and walking bass\"\n- \"epic orchestral soundtrack with soaring strings and powerful brass\"\n- \"lo-fi hip hop beat with vinyl crackle, muted keys, and chill vibes\"\n- \"acoustic folk song with fingerpicked guitar and warm harmonies\"\n\nInclude:\n- Genre (rock, jazz, electronic, classical, hip-hop, etc.)\n- Mood (happy, melancholic, energetic, relaxing, epic, etc.)\n- Instruments (piano, guitar, drums, synth, strings, etc.)\n- Style descriptors (driving, mellow, atmospheric, punchy, etc.)\n\nMODEL \"music3\": MiniMax Music 3 wants a structured caption instead of a tag list — write the prompt as one paragraph in three labeled parts: \"Global Metadata: genre, BPM, key, emotional progression across the song, production profile. Vocal Details: gender, timbre, delivery, harmonies (or: none, purely instrumental). Arrangement: primary and secondary instruments, groove, bass, percussion, textures, how sections evolve.\" The more specific, the closer the result. Fold tempo and key into this caption — music3 ignores the bpm/keyscale/timesig args.\n\nBATCH VARIATIONS: When numberOfVariations > 1, use Dynamic Prompt syntax to vary ONE dimension across separate tracks. This is one Sogni project with multiple jobs, so prefer it when all tracks share the same duration, BPM, key, lyrics, model, and generation parameters and only prompt text varies. Lock in any genre/mood/instruments the user specified, vary the rest. Example: \"{lo-fi hip hop beat with muted keys|jazz piano trio with brushed drums|ambient electronic with soft pads} with warm reverb and vinyl texture\"."
|
|
13
13
|
},
|
|
14
14
|
"duration": {
|
|
15
15
|
"type": "number",
|
|
16
|
-
"description": "Duration in seconds.
|
|
16
|
+
"description": "Duration in seconds. music3 (the default model): 10-300, default 60, and a ceiling — the song may end earlier at a musical resolution. ACE-Step turbo/sft: 10-600, default 30. Short clips: 10-30s. Standard songs: 120-300s.",
|
|
17
17
|
"minimum": 10,
|
|
18
18
|
"maximum": 600
|
|
19
19
|
},
|
|
20
20
|
"bpm": {
|
|
21
21
|
"type": "number",
|
|
22
|
-
"description": "Beats per minute / tempo. Default: 120. Range: 30-300. Slow ballad: 60-80. Mid-tempo: 90-120. Upbeat: 120-140. Fast dance: 140-180. Very fast: 180+.",
|
|
22
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the tempo in the prompt instead. Beats per minute / tempo. Default: 120. Range: 30-300. Slow ballad: 60-80. Mid-tempo: 90-120. Upbeat: 120-140. Fast dance: 140-180. Very fast: 180+.",
|
|
23
23
|
"minimum": 30,
|
|
24
24
|
"maximum": 300
|
|
25
25
|
},
|
|
26
26
|
"keyscale": {
|
|
27
27
|
"type": "string",
|
|
28
|
-
"description": "Musical key and scale. E.g., \"C major\", \"A minor\", \"F# minor\", \"Bb major\". Default: \"C major\". Only set when the user specifies a key or when a particular mood calls for it (minor keys for sad/dark, major for happy/bright)."
|
|
28
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the key in the prompt instead. Musical key and scale. E.g., \"C major\", \"A minor\", \"F# minor\", \"Bb major\". Default: \"C major\". Only set when the user specifies a key or when a particular mood calls for it (minor keys for sad/dark, major for happy/bright)."
|
|
29
29
|
},
|
|
30
30
|
"lyrics": {
|
|
31
31
|
"type": "string",
|
|
32
|
-
"description": "Song lyrics. Optional — omit for instrumental music. Format: write lyrics naturally with line breaks. The model will attempt to sing these lyrics with the generated music. Works best with clear, rhythmic phrasing that matches the BPM."
|
|
32
|
+
"description": "Song lyrics. Optional — omit for instrumental music. Format: write lyrics naturally with line breaks. The model will attempt to sing these lyrics with the generated music. Works best with clear, rhythmic phrasing that matches the BPM. For model \"music3\", structure lyrics with plain section tags on their own lines ([Intro], [Verse], [Pre-Chorus], [Chorus], [Post-Chorus], [Bridge], [Solo], [Outro]) — no modifiers inside brackets, and write enough sections to fill the requested duration since the composer ends the song when the lyric sheet runs out. For instrumental music3 tracks, pass ONLY a skeleton of those tags one per line (e.g. [Intro] [Verse] [Chorus] [Verse] [Solo] [Chorus] [Outro]) — bare instrumental pieces end early without it."
|
|
33
33
|
},
|
|
34
34
|
"model": {
|
|
35
35
|
"type": "string",
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"sft",
|
|
39
39
|
"music3"
|
|
40
40
|
],
|
|
41
|
-
"description": "Music model. \"
|
|
41
|
+
"description": "Music model. \"music3\" (default): MiniMax Music 3 — premium autoregressive composer with the best vocals, lyric adherence and song structure; 30 steps, up to 5 minutes, and it treats duration as a ceiling (may end the song early at a musical resolution). BPM/key/timesig args are ignored by music3 — fold tempo and key into the prompt instead. \"turbo\": ACE-Step 1.5 Turbo — fast 4-16 step drafts at roughly 1/20 the music3 cost; use only when the user asks for a quick, cheap, or draft track, or names ACE-Step. \"sft\": ACE-Step 1.5 SFT — experimental, strong lyric handling, 10-200 steps; use only when the user names it. Default to \"music3\" whenever the user does not ask for a draft or a specific model."
|
|
42
42
|
},
|
|
43
43
|
"timesig": {
|
|
44
44
|
"type": "number",
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
4,
|
|
49
49
|
6
|
|
50
50
|
],
|
|
51
|
-
"description": "Time signature (beats per measure). 4 = 4/4 time (default, most common). 3 = 3/4 time (waltz). 2 = 2/4 time (march). 6 = 6/8 time (compound). Default: 4."
|
|
51
|
+
"description": "ACE-Step (turbo, sft) only — music3 does not use it; for music3 put the time signature in the prompt instead. Time signature (beats per measure). 4 = 4/4 time (default, most common). 3 = 3/4 time (waltz). 2 = 2/4 time (march). 6 = 6/8 time (compound). Default: 4."
|
|
52
52
|
},
|
|
53
53
|
"numberOfVariations": {
|
|
54
54
|
"type": "number",
|
|
55
|
-
"description": "Number of variations (1-16). Use
|
|
55
|
+
"description": "Number of variations (1-16). Use with one Dynamic Prompt branch when the user requests multiple prompt-only music variations that share the same duration, BPM, key, lyrics, model, and parameters. This creates one Sogni project with multiple jobs. Default: 1.",
|
|
56
56
|
"minimum": 1,
|
|
57
57
|
"maximum": 16
|
|
58
58
|
}
|
|
@@ -86,7 +86,7 @@
|
|
|
86
86
|
"minimax-h3-fasth3-a2v-turbo",
|
|
87
87
|
"minimax-h3-fasth3-a2v-turbo-2stage"
|
|
88
88
|
],
|
|
89
|
-
"description": "\"ltx25-ia2v\" (default with image) and \"ltx25-a2v\" (default without image): LTX 2.5 image+audio and audio-only modes; Fast, HQ, and Pro currently use the release-validated official Distilled INT8 workflows. The Dev checkpoints are not publicly routed until upstream publishes and Sogni validates official ComfyUI Dev recipes. Video model. \"ltx23-ia2v\" (rollback with image): LTX 2.3 image+audio to video, audio-reactive with a reference image; Fast/HQ use the distilled 8-step worker and Default Media Quality Pro uses the non-distilled dev worker. \"ltx23-a2v\" (rollback without image): LTX 2.3 audio-only to video, no image needed, creates video purely from text prompt + audio with the same quality-tier routing. \"wan-s2v\": WAN 2.2 sound-to-video, best for lip-sync with a face image, fast 4-step. \"seedance2\": full Seedance 2.0 audio-reference video, 4-15s. \"seedance2-mini\": Seedance 2.0 Mini, 720p cap, fastest/lower-cost Seedance option. Seedance quality is selected only by this model value: pick \"seedance2-mini\" for faster/lower-cost drafts or explicit Mini requests, and pick \"seedance2\" for full-quality Seedance or 1080p/4K. Do not infer the Seedance model from Default Media Quality Fast/HQ/Pro. \"seedance2-5\": Seedance 2.5, the newest Seedance generation — 480p, 720p, and 1080p (4K is unsupported), 4-30s per clip at a fixed 24 fps, native audio, first-and-last-frame conditioning, and a much larger reference budget than the 2.0 family: up to 30 images, 10 videos, and 10 audios, with up to 50 reference media files total, subject to those per-modality caps. Choose \"seedance2-5\" when the user asks for Seedance 2.5, wants a single continuous Seedance clip longer than 15s (2.5 renders up to 30s in one call instead of being split and stitched), or wants a first-and-last-frame Seedance transition. Keep \"seedance2\" for 4K requests; Seedance 2.5 supports up to 1080p. For Seedance audio-reference prompts, preserve exact spoken dialogue when the user supplied it, and assign @Image1/@Audio1 roles. If the user asks for speech without words, describe the vocal performance without inventing quoted dialogue. Treat lip-sync, voice cloning, and real-human reference behavior as provider-sensitive rather than guaranteed. Omit to auto-select based on whether an image is present. \"wan3.0-video\" is Alibaba Wan 3 and \"wan3.0-spicy-video\" is MuleRouter w3.0-video. Both render 2-30s at fixed 30 fps with optional native audio, provider prompt expansion, 480p/720p/1080p, adaptive/fixed ratios, first/last frames, and up to 10 image/5 video/5 audio references. Only Alibaba wan3.0-video accepts document/web context and watermark. Frame anchors and loose references are mutually exclusive. Do not send negativePrompt; video references are loose conditioning for a new result, not source-video editing or extension. MiniMax H3 FastH3 audio guide: \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image from sourceImageIndex plus the audio), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame from sourceImageIndex, last frame from endImageIndex, plus the audio) and \"minimax-h3-fasth3-a2v-turbo\" (audio only; set neither sourceImageIndex nor endImageIndex) run the four-step FastH3 engine with the uploaded audio driving the picture from frame 0, and the output keeps that audio as its soundtrack. Choose them only when the user asks for MiniMax H3 or FastH3; never pick them in place of the LTX 2.5 defaults. They render 124-362 frames on the H3 17-frame grid at a fixed 24 fps (about 5.2-15.1 seconds) on a 32px grid within 1344x768, so use targetResolution 768 or omit it. audioStart picks the window of the upload. generateAudio=false
|
|
89
|
+
"description": "\"ltx25-ia2v\" (default with image) and \"ltx25-a2v\" (default without image): LTX 2.5 image+audio and audio-only modes; Fast, HQ, and Pro currently use the release-validated official Distilled INT8 workflows. The Dev checkpoints are not publicly routed until upstream publishes and Sogni validates official ComfyUI Dev recipes. Video model. \"ltx23-ia2v\" (rollback with image): LTX 2.3 image+audio to video, audio-reactive with a reference image; Fast/HQ use the distilled 8-step worker and Default Media Quality Pro uses the non-distilled dev worker. \"ltx23-a2v\" (rollback without image): LTX 2.3 audio-only to video, no image needed, creates video purely from text prompt + audio with the same quality-tier routing. \"wan-s2v\": WAN 2.2 sound-to-video, best for lip-sync with a face image, fast 4-step. \"seedance2\": full Seedance 2.0 audio-reference video, 4-15s. \"seedance2-mini\": Seedance 2.0 Mini, 720p cap, fastest/lower-cost Seedance option. Seedance quality is selected only by this model value: pick \"seedance2-mini\" for faster/lower-cost drafts or explicit Mini requests, and pick \"seedance2\" for full-quality Seedance or 1080p/4K. Do not infer the Seedance model from Default Media Quality Fast/HQ/Pro. \"seedance2-5\": Seedance 2.5, the newest Seedance generation — 480p, 720p, and 1080p (4K is unsupported), 4-30s per clip at a fixed 24 fps, native audio, first-and-last-frame conditioning, and a much larger reference budget than the 2.0 family: up to 30 images, 10 videos, and 10 audios, with up to 50 reference media files total, subject to those per-modality caps. Choose \"seedance2-5\" when the user asks for Seedance 2.5, wants a single continuous Seedance clip longer than 15s (2.5 renders up to 30s in one call instead of being split and stitched), or wants a first-and-last-frame Seedance transition. Keep \"seedance2\" for 4K requests; Seedance 2.5 supports up to 1080p. For Seedance audio-reference prompts, preserve exact spoken dialogue when the user supplied it, and assign @Image1/@Audio1 roles. If the user asks for speech without words, describe the vocal performance without inventing quoted dialogue. Treat lip-sync, voice cloning, and real-human reference behavior as provider-sensitive rather than guaranteed. Omit to auto-select based on whether an image is present. \"wan3.0-video\" is Alibaba Wan 3 and \"wan3.0-spicy-video\" is MuleRouter w3.0-video. Both render 2-30s at fixed 30 fps with optional native audio, provider prompt expansion, 480p/720p/1080p, adaptive/fixed ratios, first/last frames, and up to 10 image/5 video/5 audio references. Only Alibaba wan3.0-video accepts document/web context and watermark. Frame anchors and loose references are mutually exclusive. Do not send negativePrompt; video references are loose conditioning for a new result, not source-video editing or extension. MiniMax H3 FastH3 audio guide: \"minimax-h3-fasth3-ia2v-turbo\" (first-frame image from sourceImageIndex plus the audio), \"minimax-h3-fasth3-flfa2v-turbo\" (first frame from sourceImageIndex, last frame from endImageIndex, plus the audio) and \"minimax-h3-fasth3-a2v-turbo\" (audio only; set neither sourceImageIndex nor endImageIndex) run the four-step FastH3 engine with the uploaded audio driving the picture from frame 0, and the output keeps that audio as its soundtrack. Choose them only when the user asks for MiniMax H3 or FastH3; never pick them in place of the LTX 2.5 defaults. They render 124-362 frames on the H3 17-frame grid at a fixed 24 fps (about 5.2-15.1 seconds) on a 32px grid within 1344x768, so use targetResolution 768 or omit it. audioStart picks the window of the upload. MiniMax H3 catalog and Personal LoRAs are supported across audio modes and tiers. generateAudio=false and negativePrompt are not supported. Each costs the FastH3 price of its image-to-video, first-and-last-frame or text-to-video mode, 4 Spark per second. \"minimax-h3-fasth3-ia2v-turbo-2stage\", \"minimax-h3-fasth3-flfa2v-turbo-2stage\" and \"minimax-h3-fasth3-a2v-turbo-2stage\" are the two-stage forms: the same inputs rendered on the FastH3 canvas, then enlarged 2x and refined, delivered at twice the canvas with the same length and audio. For them targetResolution names the delivered class: 1080 renders a 544px short-edge canvas (960x544 is delivered at 1920x1088) for 10 Spark per second, and 1440 or omitted renders the 768p canvas for 2K (1344x768 is delivered at 2688x1536) for 16 Spark per second. The audio two-stage selectors have no 720p price class (720 would render a 384px canvas at the 1080p rate), so for 720p or 768p audio-guided H3 output use the regular selector. The estimate prices every request."
|
|
90
90
|
},
|
|
91
91
|
"generateAudio": {
|
|
92
92
|
"type": "boolean",
|
|
@@ -117,6 +117,25 @@
|
|
|
117
117
|
"returnLastFrame": {
|
|
118
118
|
"type": "boolean",
|
|
119
119
|
"description": "Seedance 2.5 only. Set true to export a separate image of the final frame alongside the video. The result includes lastFrameUrl, which can be used as the first-frame image for a subsequent clip. Defaults to false; this does not extend the video automatically."
|
|
120
|
+
},
|
|
121
|
+
"loras": {
|
|
122
|
+
"type": "array",
|
|
123
|
+
"minItems": 1,
|
|
124
|
+
"maxItems": 8,
|
|
125
|
+
"items": {
|
|
126
|
+
"type": "string",
|
|
127
|
+
"minLength": 1
|
|
128
|
+
},
|
|
129
|
+
"description": "Ordered MiniMax H3 LoRA IDs, including owned ready Personal LoRAs. Stack up to 8 in one request; order matters because the adapters apply in sequence and do not commute. Keep this array positionally aligned with loraStrengths. The first render with an uncached LoRA takes longer to start while the worker downloads it.\n\nAccepted only when videoModel is one of \"minimax-h3-fasth3-ia2v-turbo\", \"minimax-h3-fasth3-ia2v-turbo-2stage\", \"minimax-h3-fasth3-flfa2v-turbo\", \"minimax-h3-fasth3-flfa2v-turbo-2stage\", \"minimax-h3-fasth3-a2v-turbo\", \"minimax-h3-fasth3-a2v-turbo-2stage\". Every other video model on this tool loads no LoRAs and silently ignores these arrays, so set videoModel to an H3 mode in the same call when the user asks for one.\n\nFive LoRAs are published for MiniMax H3 today and the set differs by mode, so GET /v1/loras/comfy?modelId=<model> is authoritative for the mode in hand and carries exact ranges, maturity flags, and anything published since. h3-realism-people (fal) is a realism pass trained on live-action footage of people: it restores skin texture and pores, stray hairs, fabric weave and a fine sensor grain that the base model smooths away, and holds up in close-up. It is the only one gated on a trigger word — put r34l1sm near the FRONT of the prompt, or the render comes back as ordinary H3 with no error. h3-vbvr-video-reasoning is a prompt-adherence pass that holds the model to what was asked instead of improvising. h3-natural-face-speech (AdaptiveVision) makes people talking on camera look and sound more natural: cheeks, brows, jaw and lips move together as in real speech, and spoken English comes through clearer; use it for talking-head shots such as vlogs, podcasts, interviews and presenters. h3-better-motion (AdaptiveVision) gives people more natural, consistent body movement — weight shifts, strides, turns and gestures that follow through — for dance, sport, walking and other full-body shots. Both AdaptiveVision LoRAs work best with short, simple prompt sentences and are not validated on reference-to-video. h3-mystic-xxx-v4 is an uncensored adult fine-tune. Personal imports are discovered through authenticated GET /v1/loras/personal/catalog; use only owned ready ids with the selected model in modelIds, and respect their strength range and requirements. Do not invent ids."
|
|
130
|
+
},
|
|
131
|
+
"loraStrengths": {
|
|
132
|
+
"type": "array",
|
|
133
|
+
"minItems": 1,
|
|
134
|
+
"maxItems": 8,
|
|
135
|
+
"items": {
|
|
136
|
+
"type": "number"
|
|
137
|
+
},
|
|
138
|
+
"description": "Strength for each LoRA in loras, in the same order. Omitting the array applies 1.0 to every LoRA, which is not every LoRA's catalog default and for h3-realism-people is already at the top of its band, so send explicit values. Video LoRAs are positive-only — unlike the bipolar Krea 2 image sliders, a negative value is not an inverse effect and 0 is off. h3-realism-people takes 0-2 and its catalog default is 0.8; 0.6-1 is the usable band. It also pulls the camera in as it climbs: at 1.5 and above the shot reliably recomposes and the grade darkens, which on an image-conditioned mode can crop the subject out of the frame the user supplied. Raise it above 1 only when the user asks for more, and prefer the default when they supplied a first or last frame. h3-vbvr-video-reasoning and h3-mystic-xxx-v4 both take 0-1 and do default to 1.0, with usable bands of 0.7-1 and 0.2-1. h3-natural-face-speech and h3-better-motion take 0-1.5 and default to 0.6; their usable band is 0.4-0.8."
|
|
120
139
|
}
|
|
121
140
|
},
|
|
122
141
|
"required": [
|
package/version.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
{
|
|
2
|
-
"protocolVersion": "7.
|
|
2
|
+
"protocolVersion": "7.6.0",
|
|
3
3
|
"description": "Sogni protocol artifact version. SDKs may refuse to operate against a protocolVersion they were not built for. Bump the major when removing or renaming any schema / enum / manifest field; bump the minor when adding new optional fields or new tools; bump the patch for description / prose changes only."
|
|
4
4
|
}
|