@slatesvideo/shared 0.4.8 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/operations/index.d.ts +43 -6
- package/dist/operations/index.js +371 -51
- package/dist/prompts/model-facts.js +34 -2
- package/dist/prompts/reference-composer.d.ts +35 -0
- package/dist/prompts/reference-composer.js +81 -0
- package/dist/skills/content.js +8 -7
- package/package.json +1 -1
- package/skills/slates-content-policy.md +8 -0
- package/skills/slates-model-selection.md +31 -1
- package/skills/slates-prompting-gpt-image-2.md +40 -0
- package/skills/slates-prompting-kling-v3.md +26 -0
- package/skills/slates-prompting-lip-sync.md +22 -8
- package/skills/slates-prompting-motion-transfer.md +21 -7
- package/skills/slates-prompting-nano-banana-2.md +9 -0
- package/skills/slates-prompting-seedance.md +9 -0
package/dist/operations/index.js
CHANGED
|
@@ -111,7 +111,8 @@ export const estimateGenerationCost = {
|
|
|
111
111
|
quantity: z.number().int().min(1).max(10).optional().describe('Number of generations (default 1)'),
|
|
112
112
|
duration: z.number().int().min(3).max(15).optional().describe('Video only — seconds. Cost scales linearly; required with a video base id.'),
|
|
113
113
|
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Video only. Seedance defaults to 1080p.'),
|
|
114
|
-
resolution: z.enum(['1k', '2k', '4k']).optional().describe('Image only (default 2k).'),
|
|
114
|
+
resolution: z.enum(['1k', '2k', '3k', '4k']).optional().describe('Image only (default 2k; 3k = gpt-image-2 1440p class).'),
|
|
115
|
+
quality: z.enum(['medium', 'high']).optional().describe('gpt-image-2 only — quality tier (default medium).'),
|
|
115
116
|
sound: z.boolean().optional().describe('Veo only — audio flag changes the cost key.'),
|
|
116
117
|
seedanceFace: z.boolean().optional().describe('Seedance AI-face route (pricier key).'),
|
|
117
118
|
seedanceRealFace: z.boolean().optional().describe('Seedance consented real-face route (premium key).'),
|
|
@@ -121,11 +122,22 @@ export const estimateGenerationCost = {
|
|
|
121
122
|
const byKey = new Map(registry.models.map((m) => [m.model, m.cost_cents]));
|
|
122
123
|
// 1) exact registry cost key
|
|
123
124
|
let key = byKey.has(input.model) ? input.model : null;
|
|
124
|
-
// 2) image base id + resolution
|
|
125
|
+
// 2) image base id + resolution (+ quality for gpt-image-2)
|
|
125
126
|
if (!key) {
|
|
126
|
-
const img = ['nano-banana-2', 'flux-2-max', 'seedream-5-lite'].find((m) => m === input.model);
|
|
127
|
+
const img = ['nano-banana-2', 'nano-banana-2-lite', 'nano-banana-pro', 'gpt-image-2', 'flux-2-max', 'seedream-5-lite'].find((m) => m === input.model);
|
|
127
128
|
if (img)
|
|
128
|
-
key = imageCostKey(img, input.resolution ?? '2k');
|
|
129
|
+
key = imageCostKey(img, input.resolution ?? (img === 'nano-banana-2-lite' ? '1k' : '2k'), input.quality ?? 'medium');
|
|
130
|
+
}
|
|
131
|
+
// 2b) Kling O3 edit base id + duration (ceiled source-clip length)
|
|
132
|
+
if (!key && (input.model === 'kling-v3.0-omni-edit' || input.model === 'kling-v3.0-omni-pro-edit')) {
|
|
133
|
+
if (!input.duration) {
|
|
134
|
+
return ok({
|
|
135
|
+
requires_clarification: true,
|
|
136
|
+
missing: ['duration'],
|
|
137
|
+
message: 'Kling O3 edit bills per second of output (≈ the source clip length, rounded up) — pass duration.',
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
key = klingEditCostKey(input.model, input.duration);
|
|
129
141
|
}
|
|
130
142
|
// 3) video base id (or cost-key spelling) → the same forgiving resolver
|
|
131
143
|
// the generate op uses, so the two can never disagree about a model.
|
|
@@ -721,25 +733,33 @@ async function previewAssets(ctx, refs) {
|
|
|
721
733
|
}
|
|
722
734
|
return out;
|
|
723
735
|
}
|
|
724
|
-
// ── Generation (cloud-routed, credits-default) ──────────────────
|
|
725
736
|
// Registry cost-key for an image model+resolution. Mirrors imageCreditKey()
|
|
726
|
-
// in slate/src/shared/pricing.ts
|
|
727
|
-
//
|
|
728
|
-
|
|
737
|
+
// in slate/src/shared/pricing.ts — MUST byte-match it (the same hard rule as
|
|
738
|
+
// videoCostKey): NB2/NB Pro price per resolution, FLUX.2 Max prices per
|
|
739
|
+
// resolution (1k is the bare key), NB2 Lite/Seedream are flat, GPT Image 2 is
|
|
740
|
+
// quality × resolution-class (med|high; low deliberately not exposed).
|
|
741
|
+
function imageCostKey(model, resolution, quality = 'medium') {
|
|
729
742
|
if (model === 'flux-2-max')
|
|
730
743
|
return resolution === '1k' ? 'flux-2-max' : `flux-2-max-${resolution}`;
|
|
731
744
|
if (model === 'seedream-5-lite')
|
|
732
745
|
return 'seedream-5-lite';
|
|
746
|
+
if (model === 'nano-banana-2-lite')
|
|
747
|
+
return 'nano-banana-2-lite';
|
|
748
|
+
if (model === 'nano-banana-pro')
|
|
749
|
+
return `nano-banana-pro-${resolution}`;
|
|
750
|
+
if (model === 'gpt-image-2')
|
|
751
|
+
return `gpt-image-2-${quality === 'high' ? 'high' : 'med'}-${resolution}`;
|
|
733
752
|
return `nano-banana-2-${resolution}`;
|
|
734
753
|
}
|
|
735
754
|
export const generateImage = {
|
|
736
755
|
id: 'slates_generate_image',
|
|
737
|
-
description: 'Generate an image via Slates credits.
|
|
756
|
+
description: 'Generate an image via Slates credits. Models: nano-banana-2 (default), nano-banana-2-lite (fast/cheap drafts, 1K only), nano-banana-pro (hero-frame/typography premium), gpt-image-2 (sharp text / character sheets / grids; quality medium|high), flux-2-max (photoreal, less censored), seedream-5-lite (cheapest flat, less censored). Which model for which job: read the slates-model-selection skill. Pass projectId to save into a Slates project (recommended — asset appears live in the desktop UI). All models except nano-banana-2 REQUIRE projectId (no headless path). REQUIRED before calling: read the slates-cost-discipline skill (and the model\'s slates-prompting-* skill). You MUST pass aspectRatio and resolution explicitly (the server returns requires_clarification when missing — defaults waste credits). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. MCP/CLI generation always charges credits. No skill files installed? Call slates_get_prompting_guide with the model\'s topic (and \'slates-cost-discipline\') before first use.',
|
|
738
757
|
input: z.object({
|
|
739
758
|
prompt: z.string().min(1).max(4000),
|
|
740
|
-
model: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional().describe('Image model. Default nano-banana-2.
|
|
741
|
-
projectId: z.string().uuid().optional().describe('Save into this Slates project. Renderer refreshes live. Required for
|
|
742
|
-
resolution: z.enum(['1k', '2k', '4k']).optional().describe('Pick deliberately: 1k drafts, 2k hero shots, 4k print/final.
|
|
759
|
+
model: z.enum(['nano-banana-2', 'nano-banana-2-lite', 'nano-banana-pro', 'gpt-image-2', 'flux-2-max', 'seedream-5-lite']).optional().describe('Image model. Default nano-banana-2. Routing doctrine: slates-model-selection skill. All except nano-banana-2 require projectId.'),
|
|
760
|
+
projectId: z.string().uuid().optional().describe('Save into this Slates project. Renderer refreshes live. Required for every model except nano-banana-2.'),
|
|
761
|
+
resolution: z.enum(['1k', '2k', '3k', '4k']).optional().describe('Pick deliberately: 1k drafts, 2k hero shots, 4k print/final. nano-banana-2-lite is 1k-only. gpt-image-2 classes: 1k=1024², 2k=1080p, 3k=1440p, 4k=2160p (3k is gpt-image-2 only). Never default this.'),
|
|
762
|
+
quality: z.enum(['medium', 'high']).optional().describe('gpt-image-2 only. medium (default) = sharp text, fast, the value seat; high = max text precision + reasoning at ~4× the price. Ignored by other models.'),
|
|
743
763
|
aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Pick deliberately from the use case. Cinematic → 16:9. TikTok/Reels/Story → 9:16. IG square → 1:1. Ultra-wide → 21:9. Ask the user when ambiguous.'),
|
|
744
764
|
count: z.number().int().min(1).max(4).optional(),
|
|
745
765
|
referenceImageUrls: z.array(z.string().url()).max(14).optional().describe('Headless (no-projectId) nano-banana-2 only: up to 14 ref URLs. For projectId runs, upload refs with slates_upload_reference_image first. Always label each image\'s role in the prompt text.'),
|
|
@@ -769,7 +789,16 @@ export const generateImage = {
|
|
|
769
789
|
}
|
|
770
790
|
const resolution = input.resolution;
|
|
771
791
|
const imageModel = input.model ?? 'nano-banana-2';
|
|
772
|
-
//
|
|
792
|
+
// The '3k' class exists only on gpt-image-2 (2560×1440); other models
|
|
793
|
+
// would mis-key the registry lookup.
|
|
794
|
+
if (resolution === '3k' && imageModel !== 'gpt-image-2') {
|
|
795
|
+
return ok({
|
|
796
|
+
requires_clarification: true,
|
|
797
|
+
missing: ['resolution'],
|
|
798
|
+
message: `3k (1440p) is a gpt-image-2 resolution class — pick 1k/2k/4k for ${imageModel}.`,
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
// Only nano-banana-2 has a headless path — everything else routes through
|
|
773
802
|
// the desktop generation pipeline, which needs a project.
|
|
774
803
|
if (imageModel !== 'nano-banana-2' && !input.projectId) {
|
|
775
804
|
return ok({
|
|
@@ -810,7 +839,14 @@ export const generateImage = {
|
|
|
810
839
|
if (input.background) {
|
|
811
840
|
await ctx.desktop().requireCapability('background-generation', 'background generation');
|
|
812
841
|
}
|
|
813
|
-
|
|
842
|
+
// New-roster models need a desktop that knows them — an older desktop's
|
|
843
|
+
// allowlist would silently fall back to nano-banana-2 while we quote the
|
|
844
|
+
// new model's price.
|
|
845
|
+
if (input.projectId &&
|
|
846
|
+
(imageModel === 'gpt-image-2' || imageModel === 'nano-banana-pro' || imageModel === 'nano-banana-2-lite')) {
|
|
847
|
+
await ctx.desktop().requireCapability('image-models-v2', `${imageModel} generation`);
|
|
848
|
+
}
|
|
849
|
+
const costKey = imageCostKey(imageModel, resolution, input.quality ?? 'medium');
|
|
814
850
|
const cloud = ctx.cloud();
|
|
815
851
|
const registry = await cloud.get('/api/agent/models');
|
|
816
852
|
const entry = registry.models.find((m) => m.model === costKey);
|
|
@@ -876,6 +912,7 @@ export const generateImage = {
|
|
|
876
912
|
resolution,
|
|
877
913
|
aspectRatio: input.aspectRatio ?? '1:1',
|
|
878
914
|
count: input.count ?? 1,
|
|
915
|
+
...(imageModel === 'gpt-image-2' ? { gptQuality: input.quality ?? 'medium' } : {}),
|
|
879
916
|
...(referenceAssetIds.length > 0 ? { referenceAssetIds } : {}),
|
|
880
917
|
background: input.background,
|
|
881
918
|
});
|
|
@@ -1034,9 +1071,10 @@ export const editImage = {
|
|
|
1034
1071
|
projectId: z.string().uuid(),
|
|
1035
1072
|
sourceAssetId: z.string().uuid().describe('Image asset to edit. Must already exist in the project.'),
|
|
1036
1073
|
prompt: z.string().min(1).max(4000).describe('The edit instruction — describe the change, not the whole image.'),
|
|
1037
|
-
editModel: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional(),
|
|
1038
|
-
referenceAssetIds: z.array(z.string().uuid()).max(13).optional().describe('
|
|
1039
|
-
resolution: z.enum(['1k', '2k', '4k']).optional(),
|
|
1074
|
+
editModel: z.enum(['nano-banana-2', 'nano-banana-2-lite', 'nano-banana-pro', 'gpt-image-2', 'flux-2-max', 'seedream-5-lite']).optional(),
|
|
1075
|
+
referenceAssetIds: z.array(z.string().uuid()).max(13).optional().describe('Nano-Banana family only: extra reference images (NB Pro takes up to 13, NB2 Lite up to 3).'),
|
|
1076
|
+
resolution: z.enum(['1k', '2k', '3k', '4k']).optional().describe('3k (1440p class) is gpt-image-2 only; nano-banana-2-lite is 1k only.'),
|
|
1077
|
+
quality: z.enum(['medium', 'high']).optional().describe('gpt-image-2 only — quality tier (default medium).'),
|
|
1040
1078
|
aspectRatio: z.string().optional(),
|
|
1041
1079
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
|
|
1042
1080
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
@@ -1048,12 +1086,15 @@ export const editImage = {
|
|
|
1048
1086
|
await desktop.requireCapability('background-generation', 'background generation');
|
|
1049
1087
|
}
|
|
1050
1088
|
const editModel = input.editModel ?? 'nano-banana-2';
|
|
1051
|
-
const resolution = input.resolution ?? '2k';
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1089
|
+
const resolution = input.resolution ?? (editModel === 'nano-banana-2-lite' ? '1k' : '2k');
|
|
1090
|
+
if ((editModel === 'gpt-image-2' || editModel === 'nano-banana-pro' || editModel === 'nano-banana-2-lite')) {
|
|
1091
|
+
await desktop.requireCapability('image-models-v2', `${editModel} editing`);
|
|
1092
|
+
}
|
|
1093
|
+
// Nano-Banana family + GPT Image 2 edits charge the same key as gen;
|
|
1094
|
+
// FLUX / Seedream route to dedicated edit endpoints priced under '-edit' keys.
|
|
1095
|
+
const costKey = editModel === 'flux-2-max' || editModel === 'seedream-5-lite'
|
|
1096
|
+
? `${imageCostKey(editModel, resolution)}-edit`
|
|
1097
|
+
: imageCostKey(editModel, resolution, input.quality ?? 'medium');
|
|
1057
1098
|
const cloud = ctx.cloud();
|
|
1058
1099
|
const registry = await cloud.get('/api/agent/models');
|
|
1059
1100
|
const entry = registry.models.find((m) => m.model === costKey);
|
|
@@ -1080,6 +1121,7 @@ export const editImage = {
|
|
|
1080
1121
|
editModel,
|
|
1081
1122
|
referenceAssetIds: input.referenceAssetIds,
|
|
1082
1123
|
resolution,
|
|
1124
|
+
...(editModel === 'gpt-image-2' ? { gptQuality: input.quality ?? 'medium' } : {}),
|
|
1083
1125
|
aspectRatio: input.aspectRatio,
|
|
1084
1126
|
background: input.background,
|
|
1085
1127
|
});
|
|
@@ -1152,16 +1194,27 @@ const KLING_TIER_MAP = {
|
|
|
1152
1194
|
'kling-v3.0-std': 'kling-v3-standard',
|
|
1153
1195
|
'kling-v3.0-pro': 'kling-v3-pro',
|
|
1154
1196
|
'kling-v3.0-omni': 'kling-v3-omni',
|
|
1197
|
+
// Missing until 2026-07-05 — the fallthrough quoted a nonexistent
|
|
1198
|
+
// 'kling-v3.0-omni-pro-…' key for O3 Pro (caught by the consistency check
|
|
1199
|
+
// the day omni-pro was added to its coverage).
|
|
1200
|
+
'kling-v3.0-omni-pro': 'kling-v3-omni-pro',
|
|
1155
1201
|
};
|
|
1156
1202
|
// Exported for scripts/pricing-consistency-check.mjs (slates-api repo), which
|
|
1157
1203
|
// asserts this builder byte-matches the desktop's klingCreditKey/seedanceCreditKey.
|
|
1158
1204
|
export function videoCostKey(input) {
|
|
1159
1205
|
if (input.model.startsWith('seedance')) {
|
|
1160
|
-
// Mirrors seedanceCreditKey() in slate/src/shared/pricing.ts (face ×
|
|
1161
|
-
// AI-face route bills the `-face-` key (~45% over faceless);
|
|
1162
|
-
// real-person route bills the premium `-realface-` key (fal partner
|
|
1206
|
+
// Mirrors seedanceCreditKey() in slate/src/shared/pricing.ts (face × vref ×
|
|
1207
|
+
// res × duration). AI-face route bills the `-face-` key (~45% over faceless);
|
|
1208
|
+
// consented real-person route bills the premium `-realface-` key (fal partner
|
|
1209
|
+
// endpoint). A reference video flips to `-vref-{res}-{T}s`, T = in + out (6..30).
|
|
1163
1210
|
const res = input.videoResolution ?? '1080p';
|
|
1164
1211
|
const face = input.seedanceRealFace ? '-realface' : input.seedanceFace ? '-face' : '';
|
|
1212
|
+
const vrefSecs = input.videoRefSeconds ?? 0;
|
|
1213
|
+
if (vrefSecs > 0) {
|
|
1214
|
+
// ceil(x - 0.05) matches the server's probe rounding — quote = bill.
|
|
1215
|
+
const total = Math.min(30, Math.max(6, Math.ceil(vrefSecs - 0.05) + input.duration));
|
|
1216
|
+
return `${input.model}${face}-vref-${res}-${total}s`;
|
|
1217
|
+
}
|
|
1165
1218
|
return `${input.model}${face}-${res}-${input.duration}s`;
|
|
1166
1219
|
}
|
|
1167
1220
|
if (input.model.startsWith('veo')) {
|
|
@@ -1178,16 +1231,26 @@ export function videoCostKey(input) {
|
|
|
1178
1231
|
if (input.model.startsWith('kling-v3.0')) {
|
|
1179
1232
|
// Mirrors klingCreditKey() in slate/src/shared/pricing.ts. Kling native 4K
|
|
1180
1233
|
// bills flat-rate keys: std/pro/omni all get a `-4k` tier key, and omni-pro
|
|
1181
|
-
// shares kling-v3-omni-4k (the o3/4k endpoint has one flat rate
|
|
1234
|
+
// shares kling-v3-omni-4k (the o3/4k endpoint has one flat rate, audio
|
|
1235
|
+
// included). At 1080p AUDIO IS A KEY DIMENSION (credits = COGS × markup,
|
|
1236
|
+
// locked 2026-07-05): sound → `-audio` variant. Kling sound defaults OFF.
|
|
1182
1237
|
const tier = KLING_TIER_MAP[input.model] ?? input.model;
|
|
1183
1238
|
if (input.videoResolution === '4k') {
|
|
1184
1239
|
const tier4k = tier === 'kling-v3-omni-pro' ? 'kling-v3-omni' : tier;
|
|
1185
1240
|
return `${tier4k}-4k-${input.duration}s`;
|
|
1186
1241
|
}
|
|
1187
|
-
return `${tier}-${input.duration}s`;
|
|
1242
|
+
return `${tier}-${input.duration}s${input.sound === true ? '-audio' : ''}`;
|
|
1188
1243
|
}
|
|
1189
1244
|
throw new Error(`Unknown video model: ${input.model}`);
|
|
1190
1245
|
}
|
|
1246
|
+
// Kling O3 video-to-video edit cost key — mirrors klingEditCreditKey() in
|
|
1247
|
+
// slate/src/shared/pricing.ts (must byte-match; checked by the slates-api
|
|
1248
|
+
// pricing-consistency script). Duration is the CEILED source-clip length —
|
|
1249
|
+
// billing is per second of output ≈ source length, always rounded up.
|
|
1250
|
+
export function klingEditCostKey(model, duration) {
|
|
1251
|
+
const tier = model === 'kling-v3.0-omni-pro-edit' ? 'kling-v3-omni-pro-edit' : 'kling-v3-omni-edit';
|
|
1252
|
+
return `${tier}-${duration}s`;
|
|
1253
|
+
}
|
|
1191
1254
|
/**
|
|
1192
1255
|
* Forgiving model-id resolver. Agents routinely paste registry COST keys
|
|
1193
1256
|
* ("kling-v3-standard-8s", "seedance-2-1080p-8s") into the `model` param —
|
|
@@ -1271,7 +1334,9 @@ export const generateVideo = {
|
|
|
1271
1334
|
characterAssetIds: z.array(z.string()).optional().describe('Character sheet assets (UUIDs or badge codes) — keeps a character consistent across the shot.'),
|
|
1272
1335
|
environmentAssetIds: z.array(z.string()).optional().describe('Environment grid assets (UUIDs or badge codes) — keeps a location/setting consistent across the shot.'),
|
|
1273
1336
|
styleAssetIds: z.array(z.string()).optional().describe('Style reference assets (UUIDs or badge codes) — locks the visual style of the shot.'),
|
|
1274
|
-
videoReferenceAssetId: z.string().optional().describe('Seedance ONLY: an existing VIDEO asset (UUID or badge code) to use as a reference
|
|
1337
|
+
videoReferenceAssetId: z.string().optional().describe('Seedance ONLY: an existing VIDEO asset (UUID or badge code) to use as a reference — edit/relocate a clip, or MOTION TRANSFER (pair with a subject in ingredientAssetIds and a prompt like "the character from image 1 performs the motion from video 1"). 2-15s. Billing switches to input+output seconds (the vref key) — pass videoReferenceSeconds so the quote is right. If the clip contains a human/AI character, pair with seedanceFace=true (the default Seedance route blocks people). Ignored by Kling/Veo.'),
|
|
1338
|
+
videoReferenceSeconds: z.number().optional().describe('REQUIRED with videoReferenceAssetId: the reference clip\'s duration in seconds (from the asset listing). Feeds the vref cost key — a video-reference gen bills combined input+output seconds; the server re-derives this by probing the clip, so an understated value just gets corrected upward.'),
|
|
1339
|
+
audioReferenceAssetId: z.string().optional().describe('Seedance ONLY: an AUDIO asset (UUID or badge code), ≤15s, used as a reference — e.g. lip-sync a character to this audio ("the character in image 1 speaks the dialogue from audio 1"). No billing surcharge (Seedance audio is included). Requires at least one image or video reference alongside.'),
|
|
1275
1340
|
sound: z.boolean().optional().describe('Kling Omni / Veo / Seedance: enable audio generation. Default true.'),
|
|
1276
1341
|
audioLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('Kling Omni only — language for dialogue.'),
|
|
1277
1342
|
generateMusic: z.boolean().optional().describe('Kling Omni only — auto-generate background music.'),
|
|
@@ -1368,17 +1433,30 @@ export const generateVideo = {
|
|
|
1368
1433
|
refInputs.push({ ref: r, role: 'style' });
|
|
1369
1434
|
if (input.videoReferenceAssetId)
|
|
1370
1435
|
refInputs.push({ ref: input.videoReferenceAssetId, role: 'video reference' });
|
|
1436
|
+
if (input.audioReferenceAssetId)
|
|
1437
|
+
refInputs.push({ ref: input.audioReferenceAssetId, role: 'audio reference' });
|
|
1371
1438
|
const resolvedRefs = await resolveAssetRefs(ctx, input.projectId, refInputs.map((r) => r.ref));
|
|
1372
1439
|
const rid = (v) => v ? (resolvedRefs.get(v)?.id ?? v) : v;
|
|
1373
1440
|
const rids = (a) => a?.map((v) => resolvedRefs.get(v)?.id ?? v);
|
|
1374
1441
|
input.firstFrameAssetId = rid(input.firstFrameAssetId);
|
|
1375
1442
|
input.lastFrameAssetId = rid(input.lastFrameAssetId);
|
|
1376
1443
|
input.videoReferenceAssetId = rid(input.videoReferenceAssetId);
|
|
1444
|
+
input.audioReferenceAssetId = rid(input.audioReferenceAssetId);
|
|
1377
1445
|
input.ingredientAssetIds = rids(input.ingredientAssetIds);
|
|
1378
1446
|
input.characterAssetIds = rids(input.characterAssetIds);
|
|
1379
1447
|
input.environmentAssetIds = rids(input.environmentAssetIds);
|
|
1380
1448
|
input.styleAssetIds = rids(input.styleAssetIds);
|
|
1381
1449
|
const refEcho = describeResolvedRefs(refInputs, resolvedRefs);
|
|
1450
|
+
// A video reference bills combined input+output seconds (the vref key) —
|
|
1451
|
+
// the quote needs the clip's length. The server probes the uploaded ref
|
|
1452
|
+
// and corrects the key anyway, so this only gates quote accuracy.
|
|
1453
|
+
if (input.videoReferenceAssetId && input.model.startsWith('seedance') && !input.videoReferenceSeconds) {
|
|
1454
|
+
return ok({
|
|
1455
|
+
requires_clarification: true,
|
|
1456
|
+
missing: ['videoReferenceSeconds'],
|
|
1457
|
+
message: 'A Seedance video reference bills on combined input+output seconds. Pass videoReferenceSeconds (the reference clip\'s duration, shown in slates_list_assets) so the pre-flight quote matches the bill.',
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1382
1460
|
const cloud = ctx.cloud();
|
|
1383
1461
|
const registry = await cloud.get('/api/agent/models');
|
|
1384
1462
|
const costKey = videoCostKey({
|
|
@@ -1388,6 +1466,7 @@ export const generateVideo = {
|
|
|
1388
1466
|
sound: input.sound,
|
|
1389
1467
|
seedanceFace: input.seedanceFace,
|
|
1390
1468
|
seedanceRealFace: input.seedanceRealFace,
|
|
1469
|
+
videoRefSeconds: input.videoReferenceAssetId ? input.videoReferenceSeconds : 0,
|
|
1391
1470
|
});
|
|
1392
1471
|
// Hard consent gate, checked before any spend: the real-face route is
|
|
1393
1472
|
// consent-attested by design (the desktop enforces it too).
|
|
@@ -1482,6 +1561,7 @@ export const generateVideo = {
|
|
|
1482
1561
|
environmentAssetIds: input.environmentAssetIds ?? [],
|
|
1483
1562
|
styleAssetIds: input.styleAssetIds ?? [],
|
|
1484
1563
|
videoReferenceAssetId: input.videoReferenceAssetId,
|
|
1564
|
+
audioReferenceAssetId: input.audioReferenceAssetId,
|
|
1485
1565
|
sound: input.sound,
|
|
1486
1566
|
audioLanguage: input.audioLanguage,
|
|
1487
1567
|
generateMusic: input.generateMusic,
|
|
@@ -1530,19 +1610,27 @@ export const generateVideo = {
|
|
|
1530
1610
|
// ── Generate lip-sync ───────────────────────────────────────────
|
|
1531
1611
|
export const generateLipSync = {
|
|
1532
1612
|
id: 'slates_generate_lip_sync',
|
|
1533
|
-
description: 'Lip-sync a still image (avatar) or a video clip to audio
|
|
1613
|
+
description: 'Lip-sync a still image (avatar) or a video clip to audio. Two engines — route per slates-model-selection: (1) engine=kling (default, cheap utility lane): sourceType=video re-syncs a clip (~$0.11 / 5s); sourceType=image animates a still avatar (avatar-standard ~$0.42 / 5s; avatar-pro ~$0.86 / 5s). Audio from TTS (ttsText + ttsVoice) or an uploaded file. Always 5 seconds. (2) engine=seedance-2 (premium single-pass): the speech is generated IN the video itself — natural delivery, a video source keeps its OWN voice (native voice clone), audio included; ttsText becomes the spoken line (no voice/speed params), or an uploaded ≤15s audio file drives the speech as a reference. Seedance sources must be 2-15s videos or images; bills seedance keys (video sources bill input+output seconds — pass sourceSeconds). Faces route via seedanceFace (default true) / seedanceRealFace+realFaceConsent for real people. REQUIRED before calling: slates-cost-discipline + slates-prompting-lip-sync skills. projectId is REQUIRED.',
|
|
1534
1614
|
input: z.object({
|
|
1535
1615
|
projectId: z.string().uuid().describe('Slates project the source asset lives in. The new lip-synced video lands here.'),
|
|
1536
1616
|
sourceAssetId: z.string().uuid().describe('Asset id of the still image (avatar flow) or video clip (lip-sync flow). Must already exist in the project — use slates_upload_reference_image or slates_generate_image / slates_generate_video first if needed.'),
|
|
1537
1617
|
sourceType: z.enum(['image', 'video']).describe('"image" = animate a still portrait (avatar). "video" = re-sync an existing talking-head clip. Determines pricing — be deliberate.'),
|
|
1538
|
-
audioMethod: z.enum(['tts', 'upload']).describe('"tts" = generate speech from ttsText. "upload" = use the file at audioFilePath (absolute path on the user\'s machine).'),
|
|
1618
|
+
audioMethod: z.enum(['tts', 'upload']).describe('"tts" = generate speech from ttsText (on Seedance the line is spoken natively in the generation). "upload" = use the file at audioFilePath (absolute path on the user\'s machine; ≤15s on Seedance).'),
|
|
1539
1619
|
ttsText: z.string().min(1).max(2000).optional().describe('Required when audioMethod=tts. The exact words the avatar/clip will speak.'),
|
|
1540
|
-
ttsVoice: z.string().optional().describe('Kling voice id (e.g. "oversea_male1"). See slates-prompting-lip-sync skill for the voice catalog.'),
|
|
1541
|
-
ttsLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('TTS language. Default EN.'),
|
|
1542
|
-
ttsSpeed: z.number().min(0.5).max(2).optional().describe('TTS speech rate. Default 1.0. Range 0.5-2.0.'),
|
|
1620
|
+
ttsVoice: z.string().optional().describe('Kling engine only — voice id (e.g. "oversea_male1"). See slates-prompting-lip-sync skill for the voice catalog. Ignored on Seedance (a video source keeps its own voice; otherwise describe the voice in ttsText context).'),
|
|
1621
|
+
ttsLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('Kling engine only — TTS language. Default EN.'),
|
|
1622
|
+
ttsSpeed: z.number().min(0.5).max(2).optional().describe('Kling engine only — TTS speech rate. Default 1.0. Range 0.5-2.0.'),
|
|
1543
1623
|
audioFilePath: z.string().optional().describe('Required when audioMethod=upload. Absolute path to the audio file on the user\'s machine (mp3, wav, m4a).'),
|
|
1544
|
-
avatarModel: z.enum(['avatar-standard', 'avatar-pro']).optional().describe('
|
|
1545
|
-
klingProvider: z.enum(['fal', 'kling']).optional().describe('
|
|
1624
|
+
avatarModel: z.enum(['avatar-standard', 'avatar-pro']).optional().describe('Kling engine, image-source only. avatar-standard ($0.42/5s) for general use. avatar-pro ($0.86/5s) for sharper face fidelity.'),
|
|
1625
|
+
klingProvider: z.enum(['fal', 'kling']).optional().describe('Kling engine only — provider routing. "fal" (default) uses Slates credits. "kling" uses the user\'s own BYOK Kling key if configured.'),
|
|
1626
|
+
engine: z.enum(['kling', 'seedance-2']).optional().describe('Default kling (cheap utility). seedance-2 = premium single-pass: natural speech generated in the video, voice cloned from a video source, audio included. Credits only.'),
|
|
1627
|
+
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Seedance engine only. Default 1080p.'),
|
|
1628
|
+
aspectRatio: z.string().optional().describe('Seedance engine only. Default 16:9.'),
|
|
1629
|
+
seedanceFace: z.boolean().optional().describe('Seedance engine only — a character\'s face is in the source (default TRUE for lip-sync; the faceless route would reject it). Bills the -face key.'),
|
|
1630
|
+
seedanceRealFace: z.boolean().optional().describe('Seedance engine only — the source shows a REAL person. Premium -realface key; REQUIRES realFaceConsent=true.'),
|
|
1631
|
+
realFaceConsent: z.boolean().optional().describe('MANDATORY with seedanceRealFace — set true only after the user explicitly confirms they hold rights/consent to the likeness.'),
|
|
1632
|
+
sourceSeconds: z.number().optional().describe('Seedance engine + sourceType=video: the source clip\'s duration in seconds (from the asset listing). Feeds the vref cost key (input+output billing).'),
|
|
1633
|
+
audioSeconds: z.number().optional().describe('Seedance engine + audioMethod=upload: the audio file\'s duration in seconds — sets the output length (4-15s).'),
|
|
1546
1634
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1547
1635
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate. Required for avatar-pro.'),
|
|
1548
1636
|
}),
|
|
@@ -1561,8 +1649,41 @@ export const generateLipSync = {
|
|
|
1561
1649
|
message: 'audioMethod=upload requires audioFilePath. Pass an absolute path to the audio file on the user\'s machine.',
|
|
1562
1650
|
});
|
|
1563
1651
|
}
|
|
1652
|
+
const isSeedance = input.engine === 'seedance-2';
|
|
1564
1653
|
let costKey;
|
|
1565
|
-
|
|
1654
|
+
let seedanceDuration = 0;
|
|
1655
|
+
if (isSeedance) {
|
|
1656
|
+
// Consent gate before any spend, mirroring slates_generate_video.
|
|
1657
|
+
if (input.seedanceRealFace && !input.realFaceConsent) {
|
|
1658
|
+
return ok({
|
|
1659
|
+
requires_clarification: true,
|
|
1660
|
+
missing: ['realFaceConsent'],
|
|
1661
|
+
message: 'Real-person lip-sync needs consent: confirm with the user that they hold the rights/consent to this likeness, then retry with realFaceConsent=true.',
|
|
1662
|
+
});
|
|
1663
|
+
}
|
|
1664
|
+
if (input.sourceType === 'video' && !input.sourceSeconds) {
|
|
1665
|
+
return ok({
|
|
1666
|
+
requires_clarification: true,
|
|
1667
|
+
missing: ['sourceSeconds'],
|
|
1668
|
+
message: 'Seedance lip-sync on a video source bills combined input+output seconds. Pass sourceSeconds (the clip\'s duration from slates_list_assets, must be 2-15s).',
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
const clamp = (n) => Math.min(15, Math.max(4, Math.ceil(n)));
|
|
1672
|
+
seedanceDuration = clamp(input.sourceType === 'video' && input.sourceSeconds
|
|
1673
|
+
? input.sourceSeconds
|
|
1674
|
+
: input.audioSeconds ?? (input.ttsText ? input.ttsText.length / 13 : 5));
|
|
1675
|
+
costKey = videoCostKey({
|
|
1676
|
+
model: 'seedance-2',
|
|
1677
|
+
duration: seedanceDuration,
|
|
1678
|
+
videoResolution: input.videoResolution ?? '1080p',
|
|
1679
|
+
// Lip-sync sources are faces by definition — face route unless
|
|
1680
|
+
// explicitly disabled or escalated to realface.
|
|
1681
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1682
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1683
|
+
videoRefSeconds: input.sourceType === 'video' ? input.sourceSeconds ?? 0 : 0,
|
|
1684
|
+
});
|
|
1685
|
+
}
|
|
1686
|
+
else if (input.sourceType === 'video') {
|
|
1566
1687
|
costKey = 'kling-lip-sync-video-5s';
|
|
1567
1688
|
}
|
|
1568
1689
|
else {
|
|
@@ -1591,7 +1712,7 @@ export const generateLipSync = {
|
|
|
1591
1712
|
estimated_cents: totalCents,
|
|
1592
1713
|
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1593
1714
|
source_ref: sourceRef,
|
|
1594
|
-
message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${costKey}. ` +
|
|
1715
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} for ${isSeedance ? `${seedanceDuration}s Seedance` : '5s'} lip-sync (${costKey}). ` +
|
|
1595
1716
|
`Source: ${sourceRef}. ${audioPreview}. ` +
|
|
1596
1717
|
`Re-call with confirm=true after the user explicitly OKs the spend. ` +
|
|
1597
1718
|
`When discussing with the user, refer to the source by its code (matches the gallery badge).`,
|
|
@@ -1614,12 +1735,28 @@ export const generateLipSync = {
|
|
|
1614
1735
|
klingProvider: input.klingProvider,
|
|
1615
1736
|
estimatedCost: totalCents,
|
|
1616
1737
|
background: input.background,
|
|
1738
|
+
// Seedance engine passthrough — the desktop delegates to the seedance
|
|
1739
|
+
// ref-to-video path (vref billing, face cascade, consent gate). The
|
|
1740
|
+
// durations ride along so the desktop bills exactly what was quoted.
|
|
1741
|
+
...(isSeedance
|
|
1742
|
+
? {
|
|
1743
|
+
lipSyncEngine: 'seedance-2',
|
|
1744
|
+
duration: seedanceDuration,
|
|
1745
|
+
videoResolution: input.videoResolution,
|
|
1746
|
+
aspectRatio: input.aspectRatio,
|
|
1747
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1748
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1749
|
+
realFaceConsent: input.realFaceConsent === true,
|
|
1750
|
+
sourceDurationSeconds: input.sourceSeconds,
|
|
1751
|
+
audioDurationSeconds: input.audioSeconds,
|
|
1752
|
+
}
|
|
1753
|
+
: {}),
|
|
1617
1754
|
});
|
|
1618
1755
|
if (!result.success)
|
|
1619
1756
|
throw new Error(result.error ?? 'Lip-sync generation failed');
|
|
1620
1757
|
if (result.background) {
|
|
1621
1758
|
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1622
|
-
return backgroundSubmitted(`5s lip-sync (${costKey})`, ids, {
|
|
1759
|
+
return backgroundSubmitted(`${isSeedance ? `${seedanceDuration}s` : '5s'} lip-sync (${costKey})`, ids, {
|
|
1623
1760
|
variant: costKey,
|
|
1624
1761
|
projectId: input.projectId,
|
|
1625
1762
|
sourceAssetId: input.sourceAssetId,
|
|
@@ -1628,7 +1765,7 @@ export const generateLipSync = {
|
|
|
1628
1765
|
});
|
|
1629
1766
|
}
|
|
1630
1767
|
return {
|
|
1631
|
-
text: `Generated 5s lip-sync (${costKey}) into project ${input.projectId} ` +
|
|
1768
|
+
text: `Generated ${isSeedance ? `${seedanceDuration}s` : '5s'} lip-sync (${costKey}) into project ${input.projectId} ` +
|
|
1632
1769
|
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
1633
1770
|
(input.audioMethod === 'tts'
|
|
1634
1771
|
? `Spoken: "${(input.ttsText ?? '').slice(0, 60)}${(input.ttsText ?? '').length > 60 ? '...' : ''}"`
|
|
@@ -1649,21 +1786,58 @@ export const generateLipSync = {
|
|
|
1649
1786
|
// ── Generate motion transfer ────────────────────────────────────
|
|
1650
1787
|
export const generateMotionTransfer = {
|
|
1651
1788
|
id: 'slates_generate_motion_transfer',
|
|
1652
|
-
description: 'Transfer the motion from a reference video onto a target image character
|
|
1789
|
+
description: 'Transfer the motion from a reference video onto a target image character. Two engines — route per slates-model-selection: (1) kling-mc-std ($0.95 / 5s) / kling-mc-pro ($1.26 / 5s) — the cheap utility lane, structured skeleton/depth retargeting, always 5s. (2) motionModel=seedance-2 — the PREMIUM lane: single-pass generation with the driving clip as a native conditioning signal (better motion fidelity + native audio), prompt-driven (write what the character does, e.g. "the character from image 1 performs the exact motion from video 1"), bills input+output seconds on seedance vref keys (pass sourceVideoSeconds; driving clip must be 2-15s). Faces: seedanceFace defaults true; a REAL person needs seedanceRealFace+realFaceConsent (premium route). REQUIRED before calling: slates-cost-discipline + slates-prompting-motion-transfer skills. projectId is REQUIRED — both assets must exist in the project. All tiers hit the >$0.50 confirm gate.',
|
|
1653
1790
|
input: z.object({
|
|
1654
1791
|
projectId: z.string().uuid().describe('Slates project. Both source and target assets must live here.'),
|
|
1655
|
-
sourceVideoAssetId: z.string().uuid().describe('Asset id of the reference video — its motion will be retargeted onto the target image. Must already exist in the project.'),
|
|
1792
|
+
sourceVideoAssetId: z.string().uuid().describe('Asset id of the reference video — its motion will be retargeted onto the target image. Must already exist in the project. Seedance engine: 2-15s clips only.'),
|
|
1656
1793
|
targetImageAssetId: z.string().uuid().describe('Asset id of the target image (the character that will perform the motion). Must already exist in the project.'),
|
|
1657
|
-
motionModel: z.enum(['kling-mc-std', 'kling-mc-pro']).optional().describe('std ($0.95)
|
|
1658
|
-
characterOrientation: z.enum(['video', 'image']).optional().describe('"video" = use the source video\'s framing. "image" = use the target image\'s framing. Default video.'),
|
|
1659
|
-
prompt: z.string().optional().describe('
|
|
1660
|
-
klingProvider: z.enum(['fal', 'kling']).optional().describe('
|
|
1794
|
+
motionModel: z.enum(['kling-mc-std', 'kling-mc-pro', 'seedance-2']).optional().describe('kling-mc-std ($0.95) general motion; kling-mc-pro ($1.26) cleaner anatomy — default. seedance-2 = premium single-pass lane (prompt-driven, native audio, input+output-second billing) — pick when motion fidelity or audio matters.'),
|
|
1795
|
+
characterOrientation: z.enum(['video', 'image']).optional().describe('Kling only. "video" = use the source video\'s framing. "image" = use the target image\'s framing. Default video.'),
|
|
1796
|
+
prompt: z.string().optional().describe('Kling: optional refinement. Seedance: THE driver — describe what the character does with the motion from the clip (ordinal references: "the character from image 1 performs the motion from video 1"). A sensible default recipe is used if omitted. Read slates-prompting-motion-transfer.'),
|
|
1797
|
+
klingProvider: z.enum(['fal', 'kling']).optional().describe('Kling engine only — provider routing. "fal" (default) uses Slates credits.'),
|
|
1798
|
+
duration: z.number().int().min(4).max(15).optional().describe('Seedance engine only — output duration in seconds (4-15). Defaults to the driving clip\'s length.'),
|
|
1799
|
+
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Seedance engine only. Default 1080p.'),
|
|
1800
|
+
aspectRatio: z.string().optional().describe('Seedance engine only. Default 16:9.'),
|
|
1801
|
+
seedanceFace: z.boolean().optional().describe('Seedance engine only — a character\'s face is in the clip/image (default TRUE for motion transfer). Bills the -face key.'),
|
|
1802
|
+
seedanceRealFace: z.boolean().optional().describe('Seedance engine only — the driving clip/subject shows a REAL person. Premium -realface key; REQUIRES realFaceConsent=true.'),
|
|
1803
|
+
realFaceConsent: z.boolean().optional().describe('MANDATORY with seedanceRealFace — set true only after the user explicitly confirms they hold rights/consent to the likeness.'),
|
|
1804
|
+
sourceVideoSeconds: z.number().optional().describe('Seedance engine: the driving clip\'s duration in seconds (from the asset listing, 2-15s). Feeds the vref cost key (input+output billing).'),
|
|
1661
1805
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1662
1806
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 confirm gate. Required — both tiers exceed.'),
|
|
1663
1807
|
}),
|
|
1664
1808
|
async run(input, ctx) {
|
|
1665
1809
|
const motionModel = input.motionModel ?? 'kling-mc-pro';
|
|
1666
|
-
const
|
|
1810
|
+
const isSeedance = motionModel === 'seedance-2';
|
|
1811
|
+
let costKey;
|
|
1812
|
+
let seedanceDuration = 0;
|
|
1813
|
+
if (isSeedance) {
|
|
1814
|
+
if (input.seedanceRealFace && !input.realFaceConsent) {
|
|
1815
|
+
return ok({
|
|
1816
|
+
requires_clarification: true,
|
|
1817
|
+
missing: ['realFaceConsent'],
|
|
1818
|
+
message: 'Real-person motion transfer needs consent: confirm with the user that they hold the rights/consent to this likeness, then retry with realFaceConsent=true.',
|
|
1819
|
+
});
|
|
1820
|
+
}
|
|
1821
|
+
if (!input.sourceVideoSeconds) {
|
|
1822
|
+
return ok({
|
|
1823
|
+
requires_clarification: true,
|
|
1824
|
+
missing: ['sourceVideoSeconds'],
|
|
1825
|
+
message: 'Seedance motion transfer bills combined input+output seconds. Pass sourceVideoSeconds (the driving clip\'s duration from slates_list_assets, must be 2-15s).',
|
|
1826
|
+
});
|
|
1827
|
+
}
|
|
1828
|
+
seedanceDuration = input.duration ?? Math.min(15, Math.max(4, Math.ceil(input.sourceVideoSeconds)));
|
|
1829
|
+
costKey = videoCostKey({
|
|
1830
|
+
model: 'seedance-2',
|
|
1831
|
+
duration: seedanceDuration,
|
|
1832
|
+
videoResolution: input.videoResolution ?? '1080p',
|
|
1833
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1834
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1835
|
+
videoRefSeconds: input.sourceVideoSeconds,
|
|
1836
|
+
});
|
|
1837
|
+
}
|
|
1838
|
+
else {
|
|
1839
|
+
costKey = motionModel === 'kling-mc-std' ? 'kling-mc-std-5s' : 'kling-mc-pro-5s';
|
|
1840
|
+
}
|
|
1667
1841
|
const cloud = ctx.cloud();
|
|
1668
1842
|
const registry = await cloud.get('/api/agent/models');
|
|
1669
1843
|
const entry = registry.models.find((m) => m.model === costKey);
|
|
@@ -1687,9 +1861,9 @@ export const generateMotionTransfer = {
|
|
|
1687
1861
|
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1688
1862
|
source_ref: source,
|
|
1689
1863
|
target_ref: target,
|
|
1690
|
-
message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${motionModel}. ` +
|
|
1864
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} for ${isSeedance ? `${seedanceDuration}s Seedance motion transfer` : `5s ${motionModel}`} (${costKey}). ` +
|
|
1691
1865
|
`Transferring motion from ${source} onto ${target}. ` +
|
|
1692
|
-
`Re-call with confirm=true after the user explicitly OKs the spend, or pick kling-mc-std to save $0.31. ` +
|
|
1866
|
+
`Re-call with confirm=true after the user explicitly OKs the spend${isSeedance ? '' : ', or pick kling-mc-std to save $0.31'}. ` +
|
|
1693
1867
|
`When discussing with the user, refer to the assets by those codes — they'll match the gallery badges.`,
|
|
1694
1868
|
});
|
|
1695
1869
|
}
|
|
@@ -1707,12 +1881,27 @@ export const generateMotionTransfer = {
|
|
|
1707
1881
|
klingProvider: input.klingProvider,
|
|
1708
1882
|
estimatedCost: totalCents,
|
|
1709
1883
|
background: input.background,
|
|
1884
|
+
// Seedance engine passthrough — the desktop delegates to the seedance
|
|
1885
|
+
// ref-to-video path (vref billing, face cascade, consent gate).
|
|
1886
|
+
...(isSeedance
|
|
1887
|
+
? {
|
|
1888
|
+
duration: seedanceDuration,
|
|
1889
|
+
videoResolution: input.videoResolution,
|
|
1890
|
+
aspectRatio: input.aspectRatio,
|
|
1891
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1892
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1893
|
+
realFaceConsent: input.realFaceConsent === true,
|
|
1894
|
+
// Ride the caller-supplied clip duration through — the asset row's
|
|
1895
|
+
// duration can be null for imported clips.
|
|
1896
|
+
sourceVideoDurationSeconds: input.sourceVideoSeconds,
|
|
1897
|
+
}
|
|
1898
|
+
: {}),
|
|
1710
1899
|
});
|
|
1711
1900
|
if (!result.success)
|
|
1712
1901
|
throw new Error(result.error ?? 'Motion transfer generation failed');
|
|
1713
1902
|
if (result.background) {
|
|
1714
1903
|
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1715
|
-
return backgroundSubmitted(`5s motion transfer (${motionModel})`, ids, {
|
|
1904
|
+
return backgroundSubmitted(`${isSeedance ? `${seedanceDuration}s` : '5s'} motion transfer (${motionModel})`, ids, {
|
|
1716
1905
|
variant: costKey,
|
|
1717
1906
|
motionModel,
|
|
1718
1907
|
projectId: input.projectId,
|
|
@@ -1723,7 +1912,7 @@ export const generateMotionTransfer = {
|
|
|
1723
1912
|
});
|
|
1724
1913
|
}
|
|
1725
1914
|
return {
|
|
1726
|
-
text: `Generated 5s motion transfer (${motionModel}) into project ${input.projectId} ` +
|
|
1915
|
+
text: `Generated ${isSeedance ? `${seedanceDuration}s` : '5s'} motion transfer (${motionModel}) into project ${input.projectId} ` +
|
|
1727
1916
|
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
|
|
1728
1917
|
(input.prompt ? ` Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` : ''),
|
|
1729
1918
|
data: {
|
|
@@ -1740,6 +1929,132 @@ export const generateMotionTransfer = {
|
|
|
1740
1929
|
};
|
|
1741
1930
|
},
|
|
1742
1931
|
};
|
|
1932
|
+
// ── Edit video (Kling O3 video-to-video) ────────────────────────
|
|
1933
|
+
export const editVideo = {
|
|
1934
|
+
id: 'slates_edit_video',
|
|
1935
|
+
description: 'Edit an EXISTING video clip with one instruction via Kling O3 video-to-video edit — character swap, environment change, style transfer — in one pass, no masking. Original motion, camera, and audio are preserved; only what the prompt names changes. Use when a clip is ~90% right (fix it, don\'t re-roll it) or to AI-edit the user\'s own footage. Source clip constraints: 3–15s, 720–3840px, MP4/MOV. Cost = per second of OUTPUT (≈ clip length, rounded UP to the next second): kling-v3.0-omni-edit ≈ 19¢/s, kling-v3.0-omni-pro-edit ≈ 25¢/s. Subjects to swap IN go as characterAssetIds (frontal + angle images become Kling elements); style refs as styleAssetIds; max 4 combined. The edited clip saves as a NEW asset linked to its parent (chain edits freely). Routing: Kling edit is the default edit tool (element lock + audio intact); prefer Seedance edit/relocate only for style-transfer-heavy jobs — see slates-model-selection. Prompting: slates-prompting-kling-v3 §Edit.',
|
|
1936
|
+
input: z.object({
|
|
1937
|
+
projectId: z.string().uuid().describe('Project the source clip lives in.'),
|
|
1938
|
+
sourceVideoAssetId: z.string().describe('The VIDEO asset to edit — UUID or badge code ("VID-V3", bare "V3"); codes resolve against the project at call time. 3–15s clips only.'),
|
|
1939
|
+
prompt: z.string().min(1).max(2500).describe('The change, not the whole scene — e.g. "replace the man with @marcus", "make it a rainy night", "turn the street into a neon Tokyo alley". Mention subjects with @name; the transport compiles them to Kling\'s @ElementN notation.'),
|
|
1940
|
+
model: z.enum(['kling-v3.0-omni-edit', 'kling-v3.0-omni-pro-edit']).optional().describe('Default kling-v3.0-omni-edit. Pro (~25¢/s vs ~19¢/s) only for hero shots where fidelity matters.'),
|
|
1941
|
+
characterAssetIds: z.array(z.string()).max(4).optional().describe('Subject/element image assets to swap IN (UUIDs or badge codes). Each becomes a Kling element (@ElementN).'),
|
|
1942
|
+
styleAssetIds: z.array(z.string()).max(4).optional().describe('Style/appearance reference images (@ImageN). Max 4 combined with characterAssetIds.'),
|
|
1943
|
+
keepAudio: z.boolean().optional().describe('Preserve the original audio track (default true).'),
|
|
1944
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1945
|
+
confirm: z.boolean().optional().describe('Set true to bypass the cost confirm gate after the user OKs the spend.'),
|
|
1946
|
+
}),
|
|
1947
|
+
async run(input, ctx) {
|
|
1948
|
+
const desktop = ctx.desktop();
|
|
1949
|
+
await desktop.requireCapability('edit-video', 'video editing (Kling O3 edit)');
|
|
1950
|
+
if (input.background) {
|
|
1951
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1952
|
+
}
|
|
1953
|
+
const model = input.model ?? 'kling-v3.0-omni-edit';
|
|
1954
|
+
// Resolve refs (UUIDs or badge codes) against the project AT CALL TIME.
|
|
1955
|
+
const refInputs = [
|
|
1956
|
+
{ ref: input.sourceVideoAssetId, role: 'source clip' },
|
|
1957
|
+
...(input.characterAssetIds ?? []).map((ref) => ({ ref, role: 'subject element' })),
|
|
1958
|
+
...(input.styleAssetIds ?? []).map((ref) => ({ ref, role: 'style' })),
|
|
1959
|
+
];
|
|
1960
|
+
const resolved = await resolveAssetRefs(ctx, input.projectId, refInputs.map((r) => r.ref));
|
|
1961
|
+
const rid = (ref) => resolved.get(ref)?.id ?? ref;
|
|
1962
|
+
const sourceId = rid(input.sourceVideoAssetId);
|
|
1963
|
+
const characterAssetIds = (input.characterAssetIds ?? []).map(rid);
|
|
1964
|
+
const styleAssetIds = (input.styleAssetIds ?? []).map(rid);
|
|
1965
|
+
const refEcho = describeResolvedRefs(refInputs, resolved);
|
|
1966
|
+
if (characterAssetIds.length + styleAssetIds.length > 4) {
|
|
1967
|
+
throw new Error('Kling O3 edit takes max 4 combined subject + style references.');
|
|
1968
|
+
}
|
|
1969
|
+
// The billed key needs the clip's duration (ceiled). Read it from the
|
|
1970
|
+
// project's asset records — the desktop route independently re-validates.
|
|
1971
|
+
const { assets } = await desktop.get('/agent/assets', { projectId: input.projectId });
|
|
1972
|
+
const sourceRow = (assets ?? []).find((a) => String(a.id).toLowerCase() === sourceId.toLowerCase());
|
|
1973
|
+
if (!sourceRow)
|
|
1974
|
+
throw new Error(`Source asset not found in project: ${input.sourceVideoAssetId}`);
|
|
1975
|
+
if (sourceRow.type !== 'video')
|
|
1976
|
+
throw new Error('sourceVideoAssetId must reference a VIDEO asset.');
|
|
1977
|
+
const clipSeconds = Number(sourceRow.duration);
|
|
1978
|
+
if (!Number.isFinite(clipSeconds) || clipSeconds <= 0) {
|
|
1979
|
+
throw new Error('Source clip has no recorded duration — cannot quote the edit. Re-import the clip or pick another.');
|
|
1980
|
+
}
|
|
1981
|
+
if (clipSeconds > 15.05 || clipSeconds < 2.95) {
|
|
1982
|
+
throw new Error(`Source clip is ${clipSeconds.toFixed(1)}s — Kling O3 edit accepts 3–15s. Trim it first (agents can pre-trim on the timeline).`);
|
|
1983
|
+
}
|
|
1984
|
+
const billedSeconds = Math.min(15, Math.max(3, Math.ceil(clipSeconds - 0.05)));
|
|
1985
|
+
const costKey = klingEditCostKey(model, billedSeconds);
|
|
1986
|
+
const cloud = ctx.cloud();
|
|
1987
|
+
const registry = await cloud.get('/api/agent/models');
|
|
1988
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
1989
|
+
if (!entry)
|
|
1990
|
+
throw new Error(`Model variant not in registry: ${costKey}`);
|
|
1991
|
+
const totalCents = entry.cost_cents;
|
|
1992
|
+
// Confirm gate — look-first: preview the source clip + refs so the LLM
|
|
1993
|
+
// sees what it's editing before committing spend (mirrors generateVideo).
|
|
1994
|
+
if ((totalCents > 50 || refInputs.length > 1) && !input.confirm) {
|
|
1995
|
+
const previews = await previewAssets(ctx, [
|
|
1996
|
+
{ id: sourceId, type: 'video', role: 'source clip' },
|
|
1997
|
+
...characterAssetIds.map((id) => ({ id, type: 'image', role: 'subject element' })),
|
|
1998
|
+
...styleAssetIds.map((id) => ({ id, type: 'image', role: 'style' })),
|
|
1999
|
+
]);
|
|
2000
|
+
return {
|
|
2001
|
+
text: `Cost: $${(totalCents / 100).toFixed(2)} (${totalCents}¢) to edit a ${billedSeconds}s clip with ${model} (${costKey}). ` +
|
|
2002
|
+
`${refEcho} Re-call with confirm=true after the user explicitly OKs the spend.`,
|
|
2003
|
+
images: previews.flatMap((p) => p.images),
|
|
2004
|
+
data: {
|
|
2005
|
+
requires_confirm: true,
|
|
2006
|
+
model,
|
|
2007
|
+
variant: costKey,
|
|
2008
|
+
billed_seconds: billedSeconds,
|
|
2009
|
+
clip_seconds: clipSeconds,
|
|
2010
|
+
estimated_cents: totalCents,
|
|
2011
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
2012
|
+
references: previews.map((p) => ({ ref: p.ref, role: p.role, ...p.meta })),
|
|
2013
|
+
},
|
|
2014
|
+
};
|
|
2015
|
+
}
|
|
2016
|
+
const result = await desktop.post('/agent/generation/edit-video', {
|
|
2017
|
+
projectId: input.projectId,
|
|
2018
|
+
model,
|
|
2019
|
+
prompt: input.prompt,
|
|
2020
|
+
sourceVideoAssetId: sourceId,
|
|
2021
|
+
characterAssetIds,
|
|
2022
|
+
styleAssetIds,
|
|
2023
|
+
keepAudio: input.keepAudio !== false,
|
|
2024
|
+
background: input.background,
|
|
2025
|
+
});
|
|
2026
|
+
if (!result.success)
|
|
2027
|
+
throw new Error(result.error ?? 'Video edit failed');
|
|
2028
|
+
if (result.background) {
|
|
2029
|
+
const ids = result.generationId ? [result.generationId] : [];
|
|
2030
|
+
return backgroundSubmitted(`${billedSeconds}s video edit (${model})`, ids, {
|
|
2031
|
+
model,
|
|
2032
|
+
variant: costKey,
|
|
2033
|
+
projectId: input.projectId,
|
|
2034
|
+
sourceVideoAssetId: sourceId,
|
|
2035
|
+
cost_cents: totalCents,
|
|
2036
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
2037
|
+
}, refEcho);
|
|
2038
|
+
}
|
|
2039
|
+
return {
|
|
2040
|
+
text: `Edited ${billedSeconds}s clip saved as a new asset in project ${input.projectId} ` +
|
|
2041
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢) via ${model}. ` +
|
|
2042
|
+
`Edit: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` +
|
|
2043
|
+
(refEcho ? ` ${refEcho}` : ''),
|
|
2044
|
+
data: {
|
|
2045
|
+
model,
|
|
2046
|
+
variant: costKey,
|
|
2047
|
+
projectId: input.projectId,
|
|
2048
|
+
sourceVideoAssetId: sourceId,
|
|
2049
|
+
billed_seconds: billedSeconds,
|
|
2050
|
+
cost_cents: totalCents,
|
|
2051
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
2052
|
+
asset: result.asset,
|
|
2053
|
+
generationId: result.generationId,
|
|
2054
|
+
},
|
|
2055
|
+
};
|
|
2056
|
+
},
|
|
2057
|
+
};
|
|
1743
2058
|
// ── Generation status (background mode) ─────────────────────────
|
|
1744
2059
|
export const getGenerationStatus = {
|
|
1745
2060
|
id: 'slates_get_generation_status',
|
|
@@ -2238,6 +2553,8 @@ function resolveGuideTopic(topic) {
|
|
|
2238
2553
|
}
|
|
2239
2554
|
if (t.startsWith('nano-banana'))
|
|
2240
2555
|
return 'slates-prompting-nano-banana-2';
|
|
2556
|
+
if (t.startsWith('gpt-image') || t.startsWith('gpt image'))
|
|
2557
|
+
return 'slates-prompting-gpt-image-2';
|
|
2241
2558
|
if (t.startsWith('flux'))
|
|
2242
2559
|
return 'slates-prompting-flux-2-max';
|
|
2243
2560
|
if (t.startsWith('seedream'))
|
|
@@ -2246,6 +2563,8 @@ function resolveGuideTopic(topic) {
|
|
|
2246
2563
|
return 'slates-prompting-veo-3';
|
|
2247
2564
|
if (t.startsWith('kling-mc'))
|
|
2248
2565
|
return 'slates-prompting-motion-transfer';
|
|
2566
|
+
if (t === 'edit-video' || t === 'video-edit' || t === 'edit video' || t === 'video edit')
|
|
2567
|
+
return 'slates-prompting-kling-v3';
|
|
2249
2568
|
if (t.startsWith('kling-v3'))
|
|
2250
2569
|
return 'slates-prompting-kling-v3';
|
|
2251
2570
|
if (t.startsWith('seedance'))
|
|
@@ -2323,6 +2642,7 @@ export const ALL_OPERATIONS = [
|
|
|
2323
2642
|
generateVideo,
|
|
2324
2643
|
generateLipSync,
|
|
2325
2644
|
generateMotionTransfer,
|
|
2645
|
+
editVideo,
|
|
2326
2646
|
editImage,
|
|
2327
2647
|
getGenerationStatus,
|
|
2328
2648
|
listGenerations,
|