@slatesvideo/shared 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.js +5 -1
- package/dist/operations/index.d.ts +49 -6
- package/dist/operations/index.js +545 -72
- package/dist/prompts/model-facts.js +34 -2
- package/dist/prompts/reference-composer.d.ts +35 -0
- package/dist/prompts/reference-composer.js +81 -0
- package/dist/skills/content.js +8 -7
- package/package.json +1 -1
- package/skills/slates-content-policy.md +8 -0
- package/skills/slates-model-selection.md +33 -1
- package/skills/slates-prompting-gpt-image-2.md +40 -0
- package/skills/slates-prompting-kling-v3.md +26 -0
- package/skills/slates-prompting-lip-sync.md +22 -8
- package/skills/slates-prompting-motion-transfer.md +21 -7
- package/skills/slates-prompting-nano-banana-2.md +9 -0
- package/skills/slates-prompting-seedance.md +9 -0
package/dist/operations/index.js
CHANGED
|
@@ -34,12 +34,13 @@ const BACKGROUND_DESCRIBE = 'Submit and return immediately with generationId(s)
|
|
|
34
34
|
// Early-return shape when a generation route accepted the job in background
|
|
35
35
|
// mode ({ background: true } in the response). No inline-image fetch — the
|
|
36
36
|
// asset doesn't exist yet; the poller delivers it on completion.
|
|
37
|
-
function backgroundSubmitted(kind, ids, extra) {
|
|
37
|
+
function backgroundSubmitted(kind, ids, extra, note) {
|
|
38
38
|
const idText = ids.length > 0 ? ids.join(', ') : '(no id returned)';
|
|
39
39
|
return {
|
|
40
40
|
text: `Submitted ${kind} in the background — generationId(s): ${idText}. ` +
|
|
41
|
-
`
|
|
42
|
-
`
|
|
41
|
+
`Call slates_get_generation_status with waitSeconds: 45 (it long-polls and returns on completion — ` +
|
|
42
|
+
`never a rapid loop; video renders commonly take 1-5 minutes). Generations survive app restarts.` +
|
|
43
|
+
(note ? ` ${note}` : ''),
|
|
43
44
|
data: { generationIds: ids, status: 'processing', ...extra },
|
|
44
45
|
};
|
|
45
46
|
}
|
|
@@ -104,23 +105,76 @@ export const listAvailableModels = {
|
|
|
104
105
|
};
|
|
105
106
|
export const estimateGenerationCost = {
|
|
106
107
|
id: 'slates_estimate_generation_cost',
|
|
107
|
-
description: 'Pre-flight cost estimate. Call before any generate_* op so the user sees "this will cost N credits" up front. Pairs with the >$0.50 confirm gate.',
|
|
108
|
+
description: 'Pre-flight cost estimate. Call before any generate_* op so the user sees "this will cost N credits" up front. Takes the SAME base model ids as the generate ops (video: "seedance-2" + duration + videoResolution; image: "nano-banana-2" + resolution) — exact registry cost keys also work. Pairs with the >$0.50 confirm gate.',
|
|
108
109
|
input: z.object({
|
|
109
|
-
model: z.string().describe('
|
|
110
|
+
model: z.string().describe('Base model id as passed to the generate op (e.g. "seedance-2", "kling-v3.0-std", "nano-banana-2") or an exact registry cost key ("nano-banana-2-2k", "seedance-2-1080p-8s")'),
|
|
110
111
|
quantity: z.number().int().min(1).max(10).optional().describe('Number of generations (default 1)'),
|
|
112
|
+
duration: z.number().int().min(3).max(15).optional().describe('Video only — seconds. Cost scales linearly; required with a video base id.'),
|
|
113
|
+
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Video only. Seedance defaults to 1080p.'),
|
|
114
|
+
resolution: z.enum(['1k', '2k', '3k', '4k']).optional().describe('Image only (default 2k; 3k = gpt-image-2 1440p class).'),
|
|
115
|
+
quality: z.enum(['medium', 'high']).optional().describe('gpt-image-2 only — quality tier (default medium).'),
|
|
116
|
+
sound: z.boolean().optional().describe('Veo only — audio flag changes the cost key.'),
|
|
117
|
+
seedanceFace: z.boolean().optional().describe('Seedance AI-face route (pricier key).'),
|
|
118
|
+
seedanceRealFace: z.boolean().optional().describe('Seedance consented real-face route (premium key).'),
|
|
111
119
|
}),
|
|
112
120
|
async run(input, ctx) {
|
|
113
121
|
const registry = await ctx.cloud().get('/api/agent/models');
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
122
|
+
const byKey = new Map(registry.models.map((m) => [m.model, m.cost_cents]));
|
|
123
|
+
// 1) exact registry cost key
|
|
124
|
+
let key = byKey.has(input.model) ? input.model : null;
|
|
125
|
+
// 2) image base id + resolution (+ quality for gpt-image-2)
|
|
126
|
+
if (!key) {
|
|
127
|
+
const img = ['nano-banana-2', 'nano-banana-2-lite', 'nano-banana-pro', 'gpt-image-2', 'flux-2-max', 'seedream-5-lite'].find((m) => m === input.model);
|
|
128
|
+
if (img)
|
|
129
|
+
key = imageCostKey(img, input.resolution ?? (img === 'nano-banana-2-lite' ? '1k' : '2k'), input.quality ?? 'medium');
|
|
130
|
+
}
|
|
131
|
+
// 2b) Kling O3 edit base id + duration (ceiled source-clip length)
|
|
132
|
+
if (!key && (input.model === 'kling-v3.0-omni-edit' || input.model === 'kling-v3.0-omni-pro-edit')) {
|
|
133
|
+
if (!input.duration) {
|
|
134
|
+
return ok({
|
|
135
|
+
requires_clarification: true,
|
|
136
|
+
missing: ['duration'],
|
|
137
|
+
message: 'Kling O3 edit bills per second of output (≈ the source clip length, rounded up) — pass duration.',
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
key = klingEditCostKey(input.model, input.duration);
|
|
141
|
+
}
|
|
142
|
+
// 3) video base id (or cost-key spelling) → the same forgiving resolver
|
|
143
|
+
// the generate op uses, so the two can never disagree about a model.
|
|
144
|
+
if (!key) {
|
|
145
|
+
const resolved = resolveVideoModel(input.model);
|
|
146
|
+
if (resolved) {
|
|
147
|
+
const duration = input.duration ?? resolved.duration;
|
|
148
|
+
if (!duration) {
|
|
149
|
+
return ok({
|
|
150
|
+
requires_clarification: true,
|
|
151
|
+
missing: ['duration'],
|
|
152
|
+
message: `"${resolved.model}" cost scales with duration — pass duration (seconds) to estimate.`,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
key = videoCostKey({
|
|
156
|
+
model: resolved.model,
|
|
157
|
+
duration,
|
|
158
|
+
videoResolution: input.videoResolution ??
|
|
159
|
+
resolved.videoResolution ??
|
|
160
|
+
(resolved.model.startsWith('seedance') ? '1080p' : undefined),
|
|
161
|
+
sound: input.sound ?? resolved.sound,
|
|
162
|
+
seedanceFace: input.seedanceFace ?? resolved.seedanceFace,
|
|
163
|
+
seedanceRealFace: input.seedanceRealFace,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const cents = key != null ? byKey.get(key) : undefined;
|
|
168
|
+
if (key == null || cents == null) {
|
|
169
|
+
throw new Error(`Unknown model: ${input.model}. Pass a base id (${VIDEO_MODELS.join(' | ')} | nano-banana-2 | flux-2-max | seedream-5-lite) plus duration/resolution params, or use slates_list_available_models with a filter.`);
|
|
117
170
|
}
|
|
118
171
|
const qty = input.quantity ?? 1;
|
|
119
|
-
const totalCents =
|
|
172
|
+
const totalCents = cents * qty;
|
|
120
173
|
return ok({
|
|
121
174
|
model: input.model,
|
|
175
|
+
cost_key: key,
|
|
122
176
|
quantity: qty,
|
|
123
|
-
cost_per_cents:
|
|
177
|
+
cost_per_cents: cents,
|
|
124
178
|
total_cents: totalCents,
|
|
125
179
|
total_dollars: (totalCents / 100).toFixed(2),
|
|
126
180
|
requires_confirm: totalCents > 50,
|
|
@@ -171,6 +225,69 @@ function compactAsset(a) {
|
|
|
171
225
|
created_at: r.createdAt ?? r.created_at ?? undefined,
|
|
172
226
|
};
|
|
173
227
|
}
|
|
228
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
229
|
+
/**
|
|
230
|
+
* Resolve asset references that may be UUIDs OR badge codes ("IMG-A8",
|
|
231
|
+
* "vid-v3", bare "A8"). Codes resolve against the project's asset list AT
|
|
232
|
+
* CALL TIME — never a stale mapping from earlier in a conversation. The
|
|
233
|
+
* user creates assets in the Slates UI mid-chat; an agent that guesses or
|
|
234
|
+
* reuses a nearby UUID for a code it never looked up burns real dollars on
|
|
235
|
+
* the wrong start frame (observed live: "use A8" generated from A5).
|
|
236
|
+
* Unknown codes throw a teaching error; unknown UUIDs pass through for the
|
|
237
|
+
* desktop route to validate.
|
|
238
|
+
*/
|
|
239
|
+
async function resolveAssetRefs(ctx, projectId, refs) {
|
|
240
|
+
const out = new Map();
|
|
241
|
+
const pending = [...new Set(refs.filter((r) => typeof r === 'string' && r.trim().length > 0))];
|
|
242
|
+
if (pending.length === 0)
|
|
243
|
+
return out;
|
|
244
|
+
const { assets } = await ctx.desktop().get('/agent/assets', { projectId });
|
|
245
|
+
const rows = (assets ?? []).map(compactAsset);
|
|
246
|
+
const byId = new Map(rows.map((a) => [String(a.id).toLowerCase(), a]));
|
|
247
|
+
const byCode = new Map(rows.filter((a) => a.code).map((a) => [String(a.code).toUpperCase(), a]));
|
|
248
|
+
for (const ref of pending) {
|
|
249
|
+
const raw = ref.trim();
|
|
250
|
+
if (UUID_RE.test(raw)) {
|
|
251
|
+
const row = byId.get(raw.toLowerCase());
|
|
252
|
+
out.set(ref, {
|
|
253
|
+
id: raw,
|
|
254
|
+
code: row ? (row.code ?? null) : null,
|
|
255
|
+
label: row ? (row.label ?? null) : null,
|
|
256
|
+
});
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
let row = byCode.get(raw.toUpperCase());
|
|
260
|
+
if (!row && /^[AVS]\d+$/i.test(raw)) {
|
|
261
|
+
// Bare "A8" / "V3" / "S1" — expand to the full badge family.
|
|
262
|
+
const norm = raw.toUpperCase();
|
|
263
|
+
const prefixed = norm.startsWith('A') ? `IMG-${norm}` : norm.startsWith('V') ? `VID-${norm}` : `AUD-${norm}`;
|
|
264
|
+
row = byCode.get(prefixed);
|
|
265
|
+
}
|
|
266
|
+
if (!row) {
|
|
267
|
+
throw new Error(`No asset matching "${ref}" in this project. Codes resolve at call time — ` +
|
|
268
|
+
`call slates_list_assets (search: "${ref}") to see what actually exists, then pass the exact code or id. Never guess.`);
|
|
269
|
+
}
|
|
270
|
+
out.set(ref, {
|
|
271
|
+
id: String(row.id),
|
|
272
|
+
code: row.code ?? null,
|
|
273
|
+
label: row.label ?? null,
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
return out;
|
|
277
|
+
}
|
|
278
|
+
/** "first frame: IMG-A8 — Untitled" echo lines for generation results. */
|
|
279
|
+
function describeResolvedRefs(refInputs, resolved) {
|
|
280
|
+
if (refInputs.length === 0)
|
|
281
|
+
return '';
|
|
282
|
+
const lines = refInputs.map(({ ref, role }) => {
|
|
283
|
+
const r = resolved.get(ref);
|
|
284
|
+
if (!r)
|
|
285
|
+
return `${role}: ${ref}`;
|
|
286
|
+
const name = r.code ?? r.id;
|
|
287
|
+
return `${role}: ${name}${r.label ? ` — ${r.label}` : ''}`;
|
|
288
|
+
});
|
|
289
|
+
return `References used: ${lines.join('; ')}.`;
|
|
290
|
+
}
|
|
174
291
|
export const listAssets = {
|
|
175
292
|
id: 'slates_list_assets',
|
|
176
293
|
description: 'List assets in a Slates project as COMPACT rows (id, code, label, type) — newest first, default limit 50. Each asset carries its short code (IMG-A12 / VID-V3 / AUD-S1 — the badge the user sees on the gallery card) and label. When the user names an asset by code ("use IMG-A36 as the reference"), pass it as `search` to resolve the assetId. Always speak about assets by code + label, never by UUID. NOTE: generate_* results already return the new asset ids — do NOT call this to find an asset you just created.',
|
|
@@ -616,29 +733,37 @@ async function previewAssets(ctx, refs) {
|
|
|
616
733
|
}
|
|
617
734
|
return out;
|
|
618
735
|
}
|
|
619
|
-
// ── Generation (cloud-routed, credits-default) ──────────────────
|
|
620
736
|
// Registry cost-key for an image model+resolution. Mirrors imageCreditKey()
|
|
621
|
-
// in slate/src/shared/pricing.ts
|
|
622
|
-
//
|
|
623
|
-
|
|
737
|
+
// in slate/src/shared/pricing.ts — MUST byte-match it (the same hard rule as
|
|
738
|
+
// videoCostKey): NB2/NB Pro price per resolution, FLUX.2 Max prices per
|
|
739
|
+
// resolution (1k is the bare key), NB2 Lite/Seedream are flat, GPT Image 2 is
|
|
740
|
+
// quality × resolution-class (med|high; low deliberately not exposed).
|
|
741
|
+
function imageCostKey(model, resolution, quality = 'medium') {
|
|
624
742
|
if (model === 'flux-2-max')
|
|
625
743
|
return resolution === '1k' ? 'flux-2-max' : `flux-2-max-${resolution}`;
|
|
626
744
|
if (model === 'seedream-5-lite')
|
|
627
745
|
return 'seedream-5-lite';
|
|
746
|
+
if (model === 'nano-banana-2-lite')
|
|
747
|
+
return 'nano-banana-2-lite';
|
|
748
|
+
if (model === 'nano-banana-pro')
|
|
749
|
+
return `nano-banana-pro-${resolution}`;
|
|
750
|
+
if (model === 'gpt-image-2')
|
|
751
|
+
return `gpt-image-2-${quality === 'high' ? 'high' : 'med'}-${resolution}`;
|
|
628
752
|
return `nano-banana-2-${resolution}`;
|
|
629
753
|
}
|
|
630
754
|
export const generateImage = {
|
|
631
755
|
id: 'slates_generate_image',
|
|
632
|
-
description: 'Generate an image via Slates credits.
|
|
756
|
+
description: 'Generate an image via Slates credits. Models: nano-banana-2 (default), nano-banana-2-lite (fast/cheap drafts, 1K only), nano-banana-pro (hero-frame/typography premium), gpt-image-2 (sharp text / character sheets / grids; quality medium|high), flux-2-max (photoreal, less censored), seedream-5-lite (cheapest flat, less censored). Which model for which job: read the slates-model-selection skill. Pass projectId to save into a Slates project (recommended — asset appears live in the desktop UI). All models except nano-banana-2 REQUIRE projectId (no headless path). REQUIRED before calling: read the slates-cost-discipline skill (and the model\'s slates-prompting-* skill). You MUST pass aspectRatio and resolution explicitly (the server returns requires_clarification when missing — defaults waste credits). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. MCP/CLI generation always charges credits. No skill files installed? Call slates_get_prompting_guide with the model\'s topic (and \'slates-cost-discipline\') before first use.',
|
|
633
757
|
input: z.object({
|
|
634
758
|
prompt: z.string().min(1).max(4000),
|
|
635
|
-
model: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional().describe('Image model. Default nano-banana-2.
|
|
636
|
-
projectId: z.string().uuid().optional().describe('Save into this Slates project. Renderer refreshes live. Required for
|
|
637
|
-
resolution: z.enum(['1k', '2k', '4k']).optional().describe('Pick deliberately: 1k drafts, 2k hero shots, 4k print/final.
|
|
759
|
+
model: z.enum(['nano-banana-2', 'nano-banana-2-lite', 'nano-banana-pro', 'gpt-image-2', 'flux-2-max', 'seedream-5-lite']).optional().describe('Image model. Default nano-banana-2. Routing doctrine: slates-model-selection skill. All except nano-banana-2 require projectId.'),
|
|
760
|
+
projectId: z.string().uuid().optional().describe('Save into this Slates project. Renderer refreshes live. Required for every model except nano-banana-2.'),
|
|
761
|
+
resolution: z.enum(['1k', '2k', '3k', '4k']).optional().describe('Pick deliberately: 1k drafts, 2k hero shots, 4k print/final. nano-banana-2-lite is 1k-only. gpt-image-2 classes: 1k=1024², 2k=1080p, 3k=1440p, 4k=2160p (3k is gpt-image-2 only). Never default this.'),
|
|
762
|
+
quality: z.enum(['medium', 'high']).optional().describe('gpt-image-2 only. medium (default) = sharp text, fast, the value seat; high = max text precision + reasoning at ~4× the price. Ignored by other models.'),
|
|
638
763
|
aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Pick deliberately from the use case. Cinematic → 16:9. TikTok/Reels/Story → 9:16. IG square → 1:1. Ultra-wide → 21:9. Ask the user when ambiguous.'),
|
|
639
764
|
count: z.number().int().min(1).max(4).optional(),
|
|
640
765
|
referenceImageUrls: z.array(z.string().url()).max(14).optional().describe('Headless (no-projectId) nano-banana-2 only: up to 14 ref URLs. For projectId runs, upload refs with slates_upload_reference_image first. Always label each image\'s role in the prompt text.'),
|
|
641
|
-
referenceAssetIds: z.array(z.string()
|
|
766
|
+
referenceAssetIds: z.array(z.string()).max(14).optional().describe("Project assets to use as reference/ingredient images — asset UUIDs or badge codes (\"IMG-A8\"); codes resolve against the project at call time. Requires projectId. For nano-banana-2 up to 14 refs; FLUX/Seedream route to their edit endpoints with lower per-model caps. Label each reference's role in the prompt text."),
|
|
642
767
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
643
768
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
|
|
644
769
|
}),
|
|
@@ -664,7 +789,16 @@ export const generateImage = {
|
|
|
664
789
|
}
|
|
665
790
|
const resolution = input.resolution;
|
|
666
791
|
const imageModel = input.model ?? 'nano-banana-2';
|
|
667
|
-
//
|
|
792
|
+
// The '3k' class exists only on gpt-image-2 (2560×1440); other models
|
|
793
|
+
// would mis-key the registry lookup.
|
|
794
|
+
if (resolution === '3k' && imageModel !== 'gpt-image-2') {
|
|
795
|
+
return ok({
|
|
796
|
+
requires_clarification: true,
|
|
797
|
+
missing: ['resolution'],
|
|
798
|
+
message: `3k (1440p) is a gpt-image-2 resolution class — pick 1k/2k/4k for ${imageModel}.`,
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
// Only nano-banana-2 has a headless path — everything else routes through
|
|
668
802
|
// the desktop generation pipeline, which needs a project.
|
|
669
803
|
if (imageModel !== 'nano-banana-2' && !input.projectId) {
|
|
670
804
|
return ok({
|
|
@@ -677,7 +811,7 @@ export const generateImage = {
|
|
|
677
811
|
// disk, so a headless run has nowhere to look them up. Same for
|
|
678
812
|
// background mode: the poller (slates_get_generation_status) reads the
|
|
679
813
|
// desktop's generation records.
|
|
680
|
-
|
|
814
|
+
let referenceAssetIds = input.referenceAssetIds ?? [];
|
|
681
815
|
if (referenceAssetIds.length > 0 && !input.projectId) {
|
|
682
816
|
return ok({
|
|
683
817
|
requires_clarification: true,
|
|
@@ -692,13 +826,27 @@ export const generateImage = {
|
|
|
692
826
|
message: 'background=true routes through the desktop generation pipeline (so slates_get_generation_status can poll it) — pass a projectId, or drop background for a blocking headless run.',
|
|
693
827
|
});
|
|
694
828
|
}
|
|
829
|
+
let refEcho = '';
|
|
695
830
|
if (referenceAssetIds.length > 0) {
|
|
696
831
|
await ctx.desktop().requireCapability('image-references', 'reference images on image generation');
|
|
832
|
+
// UUIDs or badge codes — resolved against the project at call time
|
|
833
|
+
// (stale-code guessing is a real, observed failure mode).
|
|
834
|
+
const resolved = await resolveAssetRefs(ctx, input.projectId, referenceAssetIds);
|
|
835
|
+
const refInputs = referenceAssetIds.map((ref) => ({ ref, role: 'reference' }));
|
|
836
|
+
referenceAssetIds = referenceAssetIds.map((ref) => resolved.get(ref)?.id ?? ref);
|
|
837
|
+
refEcho = describeResolvedRefs(refInputs, resolved);
|
|
697
838
|
}
|
|
698
839
|
if (input.background) {
|
|
699
840
|
await ctx.desktop().requireCapability('background-generation', 'background generation');
|
|
700
841
|
}
|
|
701
|
-
|
|
842
|
+
// New-roster models need a desktop that knows them — an older desktop's
|
|
843
|
+
// allowlist would silently fall back to nano-banana-2 while we quote the
|
|
844
|
+
// new model's price.
|
|
845
|
+
if (input.projectId &&
|
|
846
|
+
(imageModel === 'gpt-image-2' || imageModel === 'nano-banana-pro' || imageModel === 'nano-banana-2-lite')) {
|
|
847
|
+
await ctx.desktop().requireCapability('image-models-v2', `${imageModel} generation`);
|
|
848
|
+
}
|
|
849
|
+
const costKey = imageCostKey(imageModel, resolution, input.quality ?? 'medium');
|
|
702
850
|
const cloud = ctx.cloud();
|
|
703
851
|
const registry = await cloud.get('/api/agent/models');
|
|
704
852
|
const entry = registry.models.find((m) => m.model === costKey);
|
|
@@ -764,6 +912,7 @@ export const generateImage = {
|
|
|
764
912
|
resolution,
|
|
765
913
|
aspectRatio: input.aspectRatio ?? '1:1',
|
|
766
914
|
count: input.count ?? 1,
|
|
915
|
+
...(imageModel === 'gpt-image-2' ? { gptQuality: input.quality ?? 'medium' } : {}),
|
|
767
916
|
...(referenceAssetIds.length > 0 ? { referenceAssetIds } : {}),
|
|
768
917
|
background: input.background,
|
|
769
918
|
});
|
|
@@ -783,7 +932,7 @@ export const generateImage = {
|
|
|
783
932
|
projectId: input.projectId,
|
|
784
933
|
cost_cents: totalCents,
|
|
785
934
|
cost_dollars: (totalCents / 100).toFixed(2),
|
|
786
|
-
});
|
|
935
|
+
}, refEcho);
|
|
787
936
|
}
|
|
788
937
|
const assetList = result.assets
|
|
789
938
|
?? (result.asset ? [result.asset] : []);
|
|
@@ -809,7 +958,8 @@ export const generateImage = {
|
|
|
809
958
|
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`
|
|
810
959
|
: `Generated ${assetList.length} image(s) into project ${input.projectId} ` +
|
|
811
960
|
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
812
|
-
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"
|
|
961
|
+
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` +
|
|
962
|
+
(refEcho ? ` ${refEcho}` : ''),
|
|
813
963
|
images,
|
|
814
964
|
data: {
|
|
815
965
|
model: imageModel,
|
|
@@ -921,9 +1071,10 @@ export const editImage = {
|
|
|
921
1071
|
projectId: z.string().uuid(),
|
|
922
1072
|
sourceAssetId: z.string().uuid().describe('Image asset to edit. Must already exist in the project.'),
|
|
923
1073
|
prompt: z.string().min(1).max(4000).describe('The edit instruction — describe the change, not the whole image.'),
|
|
924
|
-
editModel: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional(),
|
|
925
|
-
referenceAssetIds: z.array(z.string().uuid()).max(13).optional().describe('
|
|
926
|
-
resolution: z.enum(['1k', '2k', '4k']).optional(),
|
|
1074
|
+
editModel: z.enum(['nano-banana-2', 'nano-banana-2-lite', 'nano-banana-pro', 'gpt-image-2', 'flux-2-max', 'seedream-5-lite']).optional(),
|
|
1075
|
+
referenceAssetIds: z.array(z.string().uuid()).max(13).optional().describe('Nano-Banana family only: extra reference images (NB Pro takes up to 13, NB2 Lite up to 3).'),
|
|
1076
|
+
resolution: z.enum(['1k', '2k', '3k', '4k']).optional().describe('3k (1440p class) is gpt-image-2 only; nano-banana-2-lite is 1k only.'),
|
|
1077
|
+
quality: z.enum(['medium', 'high']).optional().describe('gpt-image-2 only — quality tier (default medium).'),
|
|
927
1078
|
aspectRatio: z.string().optional(),
|
|
928
1079
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
|
|
929
1080
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
@@ -935,12 +1086,15 @@ export const editImage = {
|
|
|
935
1086
|
await desktop.requireCapability('background-generation', 'background generation');
|
|
936
1087
|
}
|
|
937
1088
|
const editModel = input.editModel ?? 'nano-banana-2';
|
|
938
|
-
const resolution = input.resolution ?? '2k';
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
1089
|
+
const resolution = input.resolution ?? (editModel === 'nano-banana-2-lite' ? '1k' : '2k');
|
|
1090
|
+
if ((editModel === 'gpt-image-2' || editModel === 'nano-banana-pro' || editModel === 'nano-banana-2-lite')) {
|
|
1091
|
+
await desktop.requireCapability('image-models-v2', `${editModel} editing`);
|
|
1092
|
+
}
|
|
1093
|
+
// Nano-Banana family + GPT Image 2 edits charge the same key as gen;
|
|
1094
|
+
// FLUX / Seedream route to dedicated edit endpoints priced under '-edit' keys.
|
|
1095
|
+
const costKey = editModel === 'flux-2-max' || editModel === 'seedream-5-lite'
|
|
1096
|
+
? `${imageCostKey(editModel, resolution)}-edit`
|
|
1097
|
+
: imageCostKey(editModel, resolution, input.quality ?? 'medium');
|
|
944
1098
|
const cloud = ctx.cloud();
|
|
945
1099
|
const registry = await cloud.get('/api/agent/models');
|
|
946
1100
|
const entry = registry.models.find((m) => m.model === costKey);
|
|
@@ -967,6 +1121,7 @@ export const editImage = {
|
|
|
967
1121
|
editModel,
|
|
968
1122
|
referenceAssetIds: input.referenceAssetIds,
|
|
969
1123
|
resolution,
|
|
1124
|
+
...(editModel === 'gpt-image-2' ? { gptQuality: input.quality ?? 'medium' } : {}),
|
|
970
1125
|
aspectRatio: input.aspectRatio,
|
|
971
1126
|
background: input.background,
|
|
972
1127
|
});
|
|
@@ -1016,7 +1171,10 @@ export const editImage = {
|
|
|
1016
1171
|
},
|
|
1017
1172
|
};
|
|
1018
1173
|
// ── Generate video ──────────────────────────────────────────────
|
|
1019
|
-
|
|
1174
|
+
// Exported: the exact `model` ids slates_generate_video accepts — consumed
|
|
1175
|
+
// by the desktop Studio Agent system prompt (SSOT; never restate these ids
|
|
1176
|
+
// in prose that can drift).
|
|
1177
|
+
export const VIDEO_MODELS = [
|
|
1020
1178
|
'kling-v3.0-std',
|
|
1021
1179
|
'kling-v3.0-pro',
|
|
1022
1180
|
'kling-v3.0-omni',
|
|
@@ -1036,16 +1194,27 @@ const KLING_TIER_MAP = {
|
|
|
1036
1194
|
'kling-v3.0-std': 'kling-v3-standard',
|
|
1037
1195
|
'kling-v3.0-pro': 'kling-v3-pro',
|
|
1038
1196
|
'kling-v3.0-omni': 'kling-v3-omni',
|
|
1197
|
+
// Missing until 2026-07-05 — the fallthrough quoted a nonexistent
|
|
1198
|
+
// 'kling-v3.0-omni-pro-…' key for O3 Pro (caught by the consistency check
|
|
1199
|
+
// the day omni-pro was added to its coverage).
|
|
1200
|
+
'kling-v3.0-omni-pro': 'kling-v3-omni-pro',
|
|
1039
1201
|
};
|
|
1040
1202
|
// Exported for scripts/pricing-consistency-check.mjs (slates-api repo), which
|
|
1041
1203
|
// asserts this builder byte-matches the desktop's klingCreditKey/seedanceCreditKey.
|
|
1042
1204
|
export function videoCostKey(input) {
|
|
1043
1205
|
if (input.model.startsWith('seedance')) {
|
|
1044
|
-
// Mirrors seedanceCreditKey() in slate/src/shared/pricing.ts (face ×
|
|
1045
|
-
// AI-face route bills the `-face-` key (~45% over faceless);
|
|
1046
|
-
// real-person route bills the premium `-realface-` key (fal partner
|
|
1206
|
+
// Mirrors seedanceCreditKey() in slate/src/shared/pricing.ts (face × vref ×
|
|
1207
|
+
// res × duration). AI-face route bills the `-face-` key (~45% over faceless);
|
|
1208
|
+
// consented real-person route bills the premium `-realface-` key (fal partner
|
|
1209
|
+
// endpoint). A reference video flips to `-vref-{res}-{T}s`, T = in + out (6..30).
|
|
1047
1210
|
const res = input.videoResolution ?? '1080p';
|
|
1048
1211
|
const face = input.seedanceRealFace ? '-realface' : input.seedanceFace ? '-face' : '';
|
|
1212
|
+
const vrefSecs = input.videoRefSeconds ?? 0;
|
|
1213
|
+
if (vrefSecs > 0) {
|
|
1214
|
+
// ceil(x - 0.05) matches the server's probe rounding — quote = bill.
|
|
1215
|
+
const total = Math.min(30, Math.max(6, Math.ceil(vrefSecs - 0.05) + input.duration));
|
|
1216
|
+
return `${input.model}${face}-vref-${res}-${total}s`;
|
|
1217
|
+
}
|
|
1049
1218
|
return `${input.model}${face}-${res}-${input.duration}s`;
|
|
1050
1219
|
}
|
|
1051
1220
|
if (input.model.startsWith('veo')) {
|
|
@@ -1062,16 +1231,26 @@ export function videoCostKey(input) {
|
|
|
1062
1231
|
if (input.model.startsWith('kling-v3.0')) {
|
|
1063
1232
|
// Mirrors klingCreditKey() in slate/src/shared/pricing.ts. Kling native 4K
|
|
1064
1233
|
// bills flat-rate keys: std/pro/omni all get a `-4k` tier key, and omni-pro
|
|
1065
|
-
// shares kling-v3-omni-4k (the o3/4k endpoint has one flat rate
|
|
1234
|
+
// shares kling-v3-omni-4k (the o3/4k endpoint has one flat rate, audio
|
|
1235
|
+
// included). At 1080p AUDIO IS A KEY DIMENSION (credits = COGS × markup,
|
|
1236
|
+
// locked 2026-07-05): sound → `-audio` variant. Kling sound defaults OFF.
|
|
1066
1237
|
const tier = KLING_TIER_MAP[input.model] ?? input.model;
|
|
1067
1238
|
if (input.videoResolution === '4k') {
|
|
1068
1239
|
const tier4k = tier === 'kling-v3-omni-pro' ? 'kling-v3-omni' : tier;
|
|
1069
1240
|
return `${tier4k}-4k-${input.duration}s`;
|
|
1070
1241
|
}
|
|
1071
|
-
return `${tier}-${input.duration}s`;
|
|
1242
|
+
return `${tier}-${input.duration}s${input.sound === true ? '-audio' : ''}`;
|
|
1072
1243
|
}
|
|
1073
1244
|
throw new Error(`Unknown video model: ${input.model}`);
|
|
1074
1245
|
}
|
|
1246
|
+
// Kling O3 video-to-video edit cost key — mirrors klingEditCreditKey() in
|
|
1247
|
+
// slate/src/shared/pricing.ts (must byte-match; checked by the slates-api
|
|
1248
|
+
// pricing-consistency script). Duration is the CEILED source-clip length —
|
|
1249
|
+
// billing is per second of output ≈ source length, always rounded up.
|
|
1250
|
+
export function klingEditCostKey(model, duration) {
|
|
1251
|
+
const tier = model === 'kling-v3.0-omni-pro-edit' ? 'kling-v3-omni-pro-edit' : 'kling-v3-omni-edit';
|
|
1252
|
+
return `${tier}-${duration}s`;
|
|
1253
|
+
}
|
|
1075
1254
|
/**
|
|
1076
1255
|
* Forgiving model-id resolver. Agents routinely paste registry COST keys
|
|
1077
1256
|
* ("kling-v3-standard-8s", "seedance-2-1080p-8s") into the `model` param —
|
|
@@ -1141,21 +1320,23 @@ function promptingSkillFor(model) {
|
|
|
1141
1320
|
}
|
|
1142
1321
|
export const generateVideo = {
|
|
1143
1322
|
id: 'slates_generate_video',
|
|
1144
|
-
description: 'Generate video via Slates credits. REQUIRED before calling: read slates-model-selection (routing
|
|
1323
|
+
description: 'Generate video via Slates credits. REQUIRED before calling: read slates-model-selection (the routing doctrine), slates-cost-discipline, and the matching per-model prompting skill (slates-prompting-seedance / slates-prompting-kling-v3 / slates-prompting-veo-3) — video models prompt very differently; load them via slates_get_prompting_guide if no skill files are installed. Read slates-content-policy when the scene involves conflict, creatures, crowds, destruction, weapons, or young characters. projectId, aspectRatio, and duration are required (requires_clarification otherwise). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. Image-to-video via firstFrameAssetId; first+last frames = Veo/Seedance only; ingredients via ingredientAssetIds (Kling Omni / Seedance). Asset params take UUIDs or badge codes ("IMG-A8").',
|
|
1145
1324
|
input: z.object({
|
|
1146
1325
|
prompt: z.string().min(1).max(4000),
|
|
1147
|
-
model: z.string().describe('One of: kling-v3.0-std | kling-v3.0-pro | kling-v3.0-omni | seedance-2 | veo-3.1-fast | veo-3.1-standard. Pass the BASE id — duration and videoResolution are separate params (registry
|
|
1326
|
+
model: z.string().describe('One of: kling-v3.0-std | kling-v3.0-pro | kling-v3.0-omni | seedance-2 | veo-3.1-fast | veo-3.1-standard. Pass the BASE id — duration and videoResolution are separate params (registry cost keys like "kling-v3-standard-8s" auto-resolve). Route per the slates-model-selection skill: Kling std = general-purpose DEFAULT, Seedance 2 = premium physics/effects/hero tier, Veo = native-synced-audio niche only (16:9, 4/6/8s) — never the default. All are VIDEO-only. For per-call cost, call slates_estimate_generation_cost — never quote prices from memory.'),
|
|
1148
1327
|
projectId: z.string().uuid().optional().describe('Save into this Slates project. Strongly recommended — the desktop UI shows a progress card live and the asset appears when complete.'),
|
|
1149
1328
|
aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Veo locks to 16:9 — passing anything else will be ignored or fail. Kling/Seedance support all.'),
|
|
1150
1329
|
duration: z.number().int().min(4).max(15).optional().describe('Seconds. Kling: 5-15. Veo: 4, 6, or 8 only (4K only at 8s). Seedance: 4-15. Default 5 if omitted but always be explicit (cost scales linearly).'),
|
|
1151
1330
|
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Veo + Seedance. Seedance: 480p/720p/1080p/4K (default 1080p; 4K is native, the most expensive). Veo: 720p/1080p same price, 4K more (8s only).'),
|
|
1152
|
-
firstFrameAssetId: z.string().
|
|
1153
|
-
lastFrameAssetId: z.string().
|
|
1154
|
-
ingredientAssetIds: z.array(z.string()
|
|
1155
|
-
characterAssetIds: z.array(z.string()
|
|
1156
|
-
environmentAssetIds: z.array(z.string()
|
|
1157
|
-
styleAssetIds: z.array(z.string()
|
|
1158
|
-
videoReferenceAssetId: z.string().
|
|
1331
|
+
firstFrameAssetId: z.string().optional().describe('Starting frame for image-to-video: asset UUID or badge code ("IMG-A8") — codes resolve against the project at call time, so a code the user just spoke is always safe to pass.'),
|
|
1332
|
+
lastFrameAssetId: z.string().optional().describe('Ending frame (UUID or badge code). Veo and Seedance only. Pairs with firstFrameAssetId for guided transitions.'),
|
|
1333
|
+
ingredientAssetIds: z.array(z.string()).max(9).optional().describe('Visual reference / ingredient assets (UUIDs or badge codes) for Kling Omni or Seedance. Up to 9 (Seedance) or 4 (Kling).'),
|
|
1334
|
+
characterAssetIds: z.array(z.string()).optional().describe('Character sheet assets (UUIDs or badge codes) — keeps a character consistent across the shot.'),
|
|
1335
|
+
environmentAssetIds: z.array(z.string()).optional().describe('Environment grid assets (UUIDs or badge codes) — keeps a location/setting consistent across the shot.'),
|
|
1336
|
+
styleAssetIds: z.array(z.string()).optional().describe('Style reference assets (UUIDs or badge codes) — locks the visual style of the shot.'),
|
|
1337
|
+
videoReferenceAssetId: z.string().optional().describe('Seedance ONLY: an existing VIDEO asset (UUID or badge code) to use as a reference — edit/relocate a clip, or MOTION TRANSFER (pair with a subject in ingredientAssetIds and a prompt like "the character from image 1 performs the motion from video 1"). 2-15s. Billing switches to input+output seconds (the vref key) — pass videoReferenceSeconds so the quote is right. If the clip contains a human/AI character, pair with seedanceFace=true (the default Seedance route blocks people). Ignored by Kling/Veo.'),
|
|
1338
|
+
videoReferenceSeconds: z.number().optional().describe('REQUIRED with videoReferenceAssetId: the reference clip\'s duration in seconds (from the asset listing). Feeds the vref cost key — a video-reference gen bills combined input+output seconds; the server re-derives this by probing the clip, so an understated value just gets corrected upward.'),
|
|
1339
|
+
audioReferenceAssetId: z.string().optional().describe('Seedance ONLY: an AUDIO asset (UUID or badge code), ≤15s, used as a reference — e.g. lip-sync a character to this audio ("the character in image 1 speaks the dialogue from audio 1"). No billing surcharge (Seedance audio is included). Requires at least one image or video reference alongside.'),
|
|
1159
1340
|
sound: z.boolean().optional().describe('Kling Omni / Veo / Seedance: enable audio generation. Default true.'),
|
|
1160
1341
|
audioLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('Kling Omni only — language for dialogue.'),
|
|
1161
1342
|
generateMusic: z.boolean().optional().describe('Kling Omni only — auto-generate background music.'),
|
|
@@ -1231,6 +1412,51 @@ export const generateVideo = {
|
|
|
1231
1412
|
});
|
|
1232
1413
|
}
|
|
1233
1414
|
}
|
|
1415
|
+
// Resolve every asset reference (UUID or badge code) against the
|
|
1416
|
+
// project AT CALL TIME. Codes the user just spoke ("a8 is the one")
|
|
1417
|
+
// resolve to whatever exists NOW — including assets created in the UI
|
|
1418
|
+
// after the agent's last list call. Unknown codes throw before any
|
|
1419
|
+
// spend. The resolved code+label is echoed in every result so a wrong
|
|
1420
|
+
// start frame is visible immediately, not after $4 of render.
|
|
1421
|
+
const refInputs = [];
|
|
1422
|
+
if (input.firstFrameAssetId)
|
|
1423
|
+
refInputs.push({ ref: input.firstFrameAssetId, role: 'first frame' });
|
|
1424
|
+
if (input.lastFrameAssetId)
|
|
1425
|
+
refInputs.push({ ref: input.lastFrameAssetId, role: 'last frame' });
|
|
1426
|
+
for (const r of input.ingredientAssetIds ?? [])
|
|
1427
|
+
refInputs.push({ ref: r, role: 'ingredient' });
|
|
1428
|
+
for (const r of input.characterAssetIds ?? [])
|
|
1429
|
+
refInputs.push({ ref: r, role: 'character' });
|
|
1430
|
+
for (const r of input.environmentAssetIds ?? [])
|
|
1431
|
+
refInputs.push({ ref: r, role: 'environment' });
|
|
1432
|
+
for (const r of input.styleAssetIds ?? [])
|
|
1433
|
+
refInputs.push({ ref: r, role: 'style' });
|
|
1434
|
+
if (input.videoReferenceAssetId)
|
|
1435
|
+
refInputs.push({ ref: input.videoReferenceAssetId, role: 'video reference' });
|
|
1436
|
+
if (input.audioReferenceAssetId)
|
|
1437
|
+
refInputs.push({ ref: input.audioReferenceAssetId, role: 'audio reference' });
|
|
1438
|
+
const resolvedRefs = await resolveAssetRefs(ctx, input.projectId, refInputs.map((r) => r.ref));
|
|
1439
|
+
const rid = (v) => v ? (resolvedRefs.get(v)?.id ?? v) : v;
|
|
1440
|
+
const rids = (a) => a?.map((v) => resolvedRefs.get(v)?.id ?? v);
|
|
1441
|
+
input.firstFrameAssetId = rid(input.firstFrameAssetId);
|
|
1442
|
+
input.lastFrameAssetId = rid(input.lastFrameAssetId);
|
|
1443
|
+
input.videoReferenceAssetId = rid(input.videoReferenceAssetId);
|
|
1444
|
+
input.audioReferenceAssetId = rid(input.audioReferenceAssetId);
|
|
1445
|
+
input.ingredientAssetIds = rids(input.ingredientAssetIds);
|
|
1446
|
+
input.characterAssetIds = rids(input.characterAssetIds);
|
|
1447
|
+
input.environmentAssetIds = rids(input.environmentAssetIds);
|
|
1448
|
+
input.styleAssetIds = rids(input.styleAssetIds);
|
|
1449
|
+
const refEcho = describeResolvedRefs(refInputs, resolvedRefs);
|
|
1450
|
+
// A video reference bills combined input+output seconds (the vref key) —
|
|
1451
|
+
// the quote needs the clip's length. The server probes the uploaded ref
|
|
1452
|
+
// and corrects the key anyway, so this only gates quote accuracy.
|
|
1453
|
+
if (input.videoReferenceAssetId && input.model.startsWith('seedance') && !input.videoReferenceSeconds) {
|
|
1454
|
+
return ok({
|
|
1455
|
+
requires_clarification: true,
|
|
1456
|
+
missing: ['videoReferenceSeconds'],
|
|
1457
|
+
message: 'A Seedance video reference bills on combined input+output seconds. Pass videoReferenceSeconds (the reference clip\'s duration, shown in slates_list_assets) so the pre-flight quote matches the bill.',
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1234
1460
|
const cloud = ctx.cloud();
|
|
1235
1461
|
const registry = await cloud.get('/api/agent/models');
|
|
1236
1462
|
const costKey = videoCostKey({
|
|
@@ -1240,6 +1466,7 @@ export const generateVideo = {
|
|
|
1240
1466
|
sound: input.sound,
|
|
1241
1467
|
seedanceFace: input.seedanceFace,
|
|
1242
1468
|
seedanceRealFace: input.seedanceRealFace,
|
|
1469
|
+
videoRefSeconds: input.videoReferenceAssetId ? input.videoReferenceSeconds : 0,
|
|
1243
1470
|
});
|
|
1244
1471
|
// Hard consent gate, checked before any spend: the real-face route is
|
|
1245
1472
|
// consent-attested by design (the desktop enforces it too).
|
|
@@ -1334,6 +1561,7 @@ export const generateVideo = {
|
|
|
1334
1561
|
environmentAssetIds: input.environmentAssetIds ?? [],
|
|
1335
1562
|
styleAssetIds: input.styleAssetIds ?? [],
|
|
1336
1563
|
videoReferenceAssetId: input.videoReferenceAssetId,
|
|
1564
|
+
audioReferenceAssetId: input.audioReferenceAssetId,
|
|
1337
1565
|
sound: input.sound,
|
|
1338
1566
|
audioLanguage: input.audioLanguage,
|
|
1339
1567
|
generateMusic: input.generateMusic,
|
|
@@ -1354,12 +1582,17 @@ export const generateVideo = {
|
|
|
1354
1582
|
projectId: input.projectId,
|
|
1355
1583
|
cost_cents: totalCents,
|
|
1356
1584
|
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1357
|
-
|
|
1585
|
+
references: refInputs.map(({ ref, role }) => ({
|
|
1586
|
+
role,
|
|
1587
|
+
...(resolvedRefs.get(ref) ?? { id: ref, code: null, label: null }),
|
|
1588
|
+
})),
|
|
1589
|
+
}, refEcho);
|
|
1358
1590
|
}
|
|
1359
1591
|
return {
|
|
1360
1592
|
text: `Generated ${input.duration}s ${input.model} video into project ${input.projectId} ` +
|
|
1361
1593
|
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
1362
|
-
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"
|
|
1594
|
+
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` +
|
|
1595
|
+
(refEcho ? ` ${refEcho}` : ''),
|
|
1363
1596
|
data: {
|
|
1364
1597
|
model: input.model,
|
|
1365
1598
|
variant: costKey,
|
|
@@ -1377,19 +1610,27 @@ export const generateVideo = {
|
|
|
1377
1610
|
// ── Generate lip-sync ───────────────────────────────────────────
|
|
1378
1611
|
export const generateLipSync = {
|
|
1379
1612
|
id: 'slates_generate_lip_sync',
|
|
1380
|
-
description: 'Lip-sync a still image (avatar) or a video clip to audio
|
|
1613
|
+
description: 'Lip-sync a still image (avatar) or a video clip to audio. Two engines — route per slates-model-selection: (1) engine=kling (default, cheap utility lane): sourceType=video re-syncs a clip (~$0.11 / 5s); sourceType=image animates a still avatar (avatar-standard ~$0.42 / 5s; avatar-pro ~$0.86 / 5s). Audio from TTS (ttsText + ttsVoice) or an uploaded file. Always 5 seconds. (2) engine=seedance-2 (premium single-pass): the speech is generated IN the video itself — natural delivery, a video source keeps its OWN voice (native voice clone), audio included; ttsText becomes the spoken line (no voice/speed params), or an uploaded ≤15s audio file drives the speech as a reference. Seedance sources must be 2-15s videos or images; bills seedance keys (video sources bill input+output seconds — pass sourceSeconds). Faces route via seedanceFace (default true) / seedanceRealFace+realFaceConsent for real people. REQUIRED before calling: slates-cost-discipline + slates-prompting-lip-sync skills. projectId is REQUIRED.',
|
|
1381
1614
|
input: z.object({
|
|
1382
1615
|
projectId: z.string().uuid().describe('Slates project the source asset lives in. The new lip-synced video lands here.'),
|
|
1383
1616
|
sourceAssetId: z.string().uuid().describe('Asset id of the still image (avatar flow) or video clip (lip-sync flow). Must already exist in the project — use slates_upload_reference_image or slates_generate_image / slates_generate_video first if needed.'),
|
|
1384
1617
|
sourceType: z.enum(['image', 'video']).describe('"image" = animate a still portrait (avatar). "video" = re-sync an existing talking-head clip. Determines pricing — be deliberate.'),
|
|
1385
|
-
audioMethod: z.enum(['tts', 'upload']).describe('"tts" = generate speech from ttsText. "upload" = use the file at audioFilePath (absolute path on the user\'s machine).'),
|
|
1618
|
+
audioMethod: z.enum(['tts', 'upload']).describe('"tts" = generate speech from ttsText (on Seedance the line is spoken natively in the generation). "upload" = use the file at audioFilePath (absolute path on the user\'s machine; ≤15s on Seedance).'),
|
|
1386
1619
|
ttsText: z.string().min(1).max(2000).optional().describe('Required when audioMethod=tts. The exact words the avatar/clip will speak.'),
|
|
1387
|
-
ttsVoice: z.string().optional().describe('Kling voice id (e.g. "oversea_male1"). See slates-prompting-lip-sync skill for the voice catalog.'),
|
|
1388
|
-
ttsLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('TTS language. Default EN.'),
|
|
1389
|
-
ttsSpeed: z.number().min(0.5).max(2).optional().describe('TTS speech rate. Default 1.0. Range 0.5-2.0.'),
|
|
1620
|
+
ttsVoice: z.string().optional().describe('Kling engine only — voice id (e.g. "oversea_male1"). See slates-prompting-lip-sync skill for the voice catalog. Ignored on Seedance (a video source keeps its own voice; otherwise describe the voice in ttsText context).'),
|
|
1621
|
+
ttsLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('Kling engine only — TTS language. Default EN.'),
|
|
1622
|
+
ttsSpeed: z.number().min(0.5).max(2).optional().describe('Kling engine only — TTS speech rate. Default 1.0. Range 0.5-2.0.'),
|
|
1390
1623
|
audioFilePath: z.string().optional().describe('Required when audioMethod=upload. Absolute path to the audio file on the user\'s machine (mp3, wav, m4a).'),
|
|
1391
|
-
avatarModel: z.enum(['avatar-standard', 'avatar-pro']).optional().describe('
|
|
1392
|
-
klingProvider: z.enum(['fal', 'kling']).optional().describe('
|
|
1624
|
+
avatarModel: z.enum(['avatar-standard', 'avatar-pro']).optional().describe('Kling engine, image-source only. avatar-standard ($0.42/5s) for general use. avatar-pro ($0.86/5s) for sharper face fidelity.'),
|
|
1625
|
+
klingProvider: z.enum(['fal', 'kling']).optional().describe('Kling engine only — provider routing. "fal" (default) uses Slates credits. "kling" uses the user\'s own BYOK Kling key if configured.'),
|
|
1626
|
+
engine: z.enum(['kling', 'seedance-2']).optional().describe('Default kling (cheap utility). seedance-2 = premium single-pass: natural speech generated in the video, voice cloned from a video source, audio included. Credits only.'),
|
|
1627
|
+
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Seedance engine only. Default 1080p.'),
|
|
1628
|
+
aspectRatio: z.string().optional().describe('Seedance engine only. Default 16:9.'),
|
|
1629
|
+
seedanceFace: z.boolean().optional().describe('Seedance engine only — a character\'s face is in the source (default TRUE for lip-sync; the faceless route would reject it). Bills the -face key.'),
|
|
1630
|
+
seedanceRealFace: z.boolean().optional().describe('Seedance engine only — the source shows a REAL person. Premium -realface key; REQUIRES realFaceConsent=true.'),
|
|
1631
|
+
realFaceConsent: z.boolean().optional().describe('MANDATORY with seedanceRealFace — set true only after the user explicitly confirms they hold rights/consent to the likeness.'),
|
|
1632
|
+
sourceSeconds: z.number().optional().describe('Seedance engine + sourceType=video: the source clip\'s duration in seconds (from the asset listing). Feeds the vref cost key (input+output billing).'),
|
|
1633
|
+
audioSeconds: z.number().optional().describe('Seedance engine + audioMethod=upload: the audio file\'s duration in seconds — sets the output length (4-15s).'),
|
|
1393
1634
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1394
1635
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate. Required for avatar-pro.'),
|
|
1395
1636
|
}),
|
|
@@ -1408,8 +1649,41 @@ export const generateLipSync = {
|
|
|
1408
1649
|
message: 'audioMethod=upload requires audioFilePath. Pass an absolute path to the audio file on the user\'s machine.',
|
|
1409
1650
|
});
|
|
1410
1651
|
}
|
|
1652
|
+
const isSeedance = input.engine === 'seedance-2';
|
|
1411
1653
|
let costKey;
|
|
1412
|
-
|
|
1654
|
+
let seedanceDuration = 0;
|
|
1655
|
+
if (isSeedance) {
|
|
1656
|
+
// Consent gate before any spend, mirroring slates_generate_video.
|
|
1657
|
+
if (input.seedanceRealFace && !input.realFaceConsent) {
|
|
1658
|
+
return ok({
|
|
1659
|
+
requires_clarification: true,
|
|
1660
|
+
missing: ['realFaceConsent'],
|
|
1661
|
+
message: 'Real-person lip-sync needs consent: confirm with the user that they hold the rights/consent to this likeness, then retry with realFaceConsent=true.',
|
|
1662
|
+
});
|
|
1663
|
+
}
|
|
1664
|
+
if (input.sourceType === 'video' && !input.sourceSeconds) {
|
|
1665
|
+
return ok({
|
|
1666
|
+
requires_clarification: true,
|
|
1667
|
+
missing: ['sourceSeconds'],
|
|
1668
|
+
message: 'Seedance lip-sync on a video source bills combined input+output seconds. Pass sourceSeconds (the clip\'s duration from slates_list_assets, must be 2-15s).',
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
const clamp = (n) => Math.min(15, Math.max(4, Math.ceil(n)));
|
|
1672
|
+
seedanceDuration = clamp(input.sourceType === 'video' && input.sourceSeconds
|
|
1673
|
+
? input.sourceSeconds
|
|
1674
|
+
: input.audioSeconds ?? (input.ttsText ? input.ttsText.length / 13 : 5));
|
|
1675
|
+
costKey = videoCostKey({
|
|
1676
|
+
model: 'seedance-2',
|
|
1677
|
+
duration: seedanceDuration,
|
|
1678
|
+
videoResolution: input.videoResolution ?? '1080p',
|
|
1679
|
+
// Lip-sync sources are faces by definition — face route unless
|
|
1680
|
+
// explicitly disabled or escalated to realface.
|
|
1681
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1682
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1683
|
+
videoRefSeconds: input.sourceType === 'video' ? input.sourceSeconds ?? 0 : 0,
|
|
1684
|
+
});
|
|
1685
|
+
}
|
|
1686
|
+
else if (input.sourceType === 'video') {
|
|
1413
1687
|
costKey = 'kling-lip-sync-video-5s';
|
|
1414
1688
|
}
|
|
1415
1689
|
else {
|
|
@@ -1438,7 +1712,7 @@ export const generateLipSync = {
|
|
|
1438
1712
|
estimated_cents: totalCents,
|
|
1439
1713
|
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1440
1714
|
source_ref: sourceRef,
|
|
1441
|
-
message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${costKey}. ` +
|
|
1715
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} for ${isSeedance ? `${seedanceDuration}s Seedance` : '5s'} lip-sync (${costKey}). ` +
|
|
1442
1716
|
`Source: ${sourceRef}. ${audioPreview}. ` +
|
|
1443
1717
|
`Re-call with confirm=true after the user explicitly OKs the spend. ` +
|
|
1444
1718
|
`When discussing with the user, refer to the source by its code (matches the gallery badge).`,
|
|
@@ -1461,12 +1735,28 @@ export const generateLipSync = {
|
|
|
1461
1735
|
klingProvider: input.klingProvider,
|
|
1462
1736
|
estimatedCost: totalCents,
|
|
1463
1737
|
background: input.background,
|
|
1738
|
+
// Seedance engine passthrough — the desktop delegates to the seedance
|
|
1739
|
+
// ref-to-video path (vref billing, face cascade, consent gate). The
|
|
1740
|
+
// durations ride along so the desktop bills exactly what was quoted.
|
|
1741
|
+
...(isSeedance
|
|
1742
|
+
? {
|
|
1743
|
+
lipSyncEngine: 'seedance-2',
|
|
1744
|
+
duration: seedanceDuration,
|
|
1745
|
+
videoResolution: input.videoResolution,
|
|
1746
|
+
aspectRatio: input.aspectRatio,
|
|
1747
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1748
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1749
|
+
realFaceConsent: input.realFaceConsent === true,
|
|
1750
|
+
sourceDurationSeconds: input.sourceSeconds,
|
|
1751
|
+
audioDurationSeconds: input.audioSeconds,
|
|
1752
|
+
}
|
|
1753
|
+
: {}),
|
|
1464
1754
|
});
|
|
1465
1755
|
if (!result.success)
|
|
1466
1756
|
throw new Error(result.error ?? 'Lip-sync generation failed');
|
|
1467
1757
|
if (result.background) {
|
|
1468
1758
|
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1469
|
-
return backgroundSubmitted(`5s lip-sync (${costKey})`, ids, {
|
|
1759
|
+
return backgroundSubmitted(`${isSeedance ? `${seedanceDuration}s` : '5s'} lip-sync (${costKey})`, ids, {
|
|
1470
1760
|
variant: costKey,
|
|
1471
1761
|
projectId: input.projectId,
|
|
1472
1762
|
sourceAssetId: input.sourceAssetId,
|
|
@@ -1475,7 +1765,7 @@ export const generateLipSync = {
|
|
|
1475
1765
|
});
|
|
1476
1766
|
}
|
|
1477
1767
|
return {
|
|
1478
|
-
text: `Generated 5s lip-sync (${costKey}) into project ${input.projectId} ` +
|
|
1768
|
+
text: `Generated ${isSeedance ? `${seedanceDuration}s` : '5s'} lip-sync (${costKey}) into project ${input.projectId} ` +
|
|
1479
1769
|
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
1480
1770
|
(input.audioMethod === 'tts'
|
|
1481
1771
|
? `Spoken: "${(input.ttsText ?? '').slice(0, 60)}${(input.ttsText ?? '').length > 60 ? '...' : ''}"`
|
|
@@ -1496,21 +1786,58 @@ export const generateLipSync = {
|
|
|
1496
1786
|
// ── Generate motion transfer ────────────────────────────────────
|
|
1497
1787
|
export const generateMotionTransfer = {
|
|
1498
1788
|
id: 'slates_generate_motion_transfer',
|
|
1499
|
-
description: 'Transfer the motion from a reference video onto a target image character
|
|
1789
|
+
description: 'Transfer the motion from a reference video onto a target image character. Two engines — route per slates-model-selection: (1) kling-mc-std ($0.95 / 5s) / kling-mc-pro ($1.26 / 5s) — the cheap utility lane, structured skeleton/depth retargeting, always 5s. (2) motionModel=seedance-2 — the PREMIUM lane: single-pass generation with the driving clip as a native conditioning signal (better motion fidelity + native audio), prompt-driven (write what the character does, e.g. "the character from image 1 performs the exact motion from video 1"), bills input+output seconds on seedance vref keys (pass sourceVideoSeconds; driving clip must be 2-15s). Faces: seedanceFace defaults true; a REAL person needs seedanceRealFace+realFaceConsent (premium route). REQUIRED before calling: slates-cost-discipline + slates-prompting-motion-transfer skills. projectId is REQUIRED — both assets must exist in the project. All tiers hit the >$0.50 confirm gate.',
|
|
1500
1790
|
input: z.object({
|
|
1501
1791
|
projectId: z.string().uuid().describe('Slates project. Both source and target assets must live here.'),
|
|
1502
|
-
sourceVideoAssetId: z.string().uuid().describe('Asset id of the reference video — its motion will be retargeted onto the target image. Must already exist in the project.'),
|
|
1792
|
+
sourceVideoAssetId: z.string().uuid().describe('Asset id of the reference video — its motion will be retargeted onto the target image. Must already exist in the project. Seedance engine: 2-15s clips only.'),
|
|
1503
1793
|
targetImageAssetId: z.string().uuid().describe('Asset id of the target image (the character that will perform the motion). Must already exist in the project.'),
|
|
1504
|
-
motionModel: z.enum(['kling-mc-std', 'kling-mc-pro']).optional().describe('std ($0.95)
|
|
1505
|
-
characterOrientation: z.enum(['video', 'image']).optional().describe('"video" = use the source video\'s framing. "image" = use the target image\'s framing. Default video.'),
|
|
1506
|
-
prompt: z.string().optional().describe('
|
|
1507
|
-
klingProvider: z.enum(['fal', 'kling']).optional().describe('
|
|
1794
|
+
motionModel: z.enum(['kling-mc-std', 'kling-mc-pro', 'seedance-2']).optional().describe('kling-mc-std ($0.95) general motion; kling-mc-pro ($1.26) cleaner anatomy — default. seedance-2 = premium single-pass lane (prompt-driven, native audio, input+output-second billing) — pick when motion fidelity or audio matters.'),
|
|
1795
|
+
characterOrientation: z.enum(['video', 'image']).optional().describe('Kling only. "video" = use the source video\'s framing. "image" = use the target image\'s framing. Default video.'),
|
|
1796
|
+
prompt: z.string().optional().describe('Kling: optional refinement. Seedance: THE driver — describe what the character does with the motion from the clip (ordinal references: "the character from image 1 performs the motion from video 1"). A sensible default recipe is used if omitted. Read slates-prompting-motion-transfer.'),
|
|
1797
|
+
klingProvider: z.enum(['fal', 'kling']).optional().describe('Kling engine only — provider routing. "fal" (default) uses Slates credits.'),
|
|
1798
|
+
duration: z.number().int().min(4).max(15).optional().describe('Seedance engine only — output duration in seconds (4-15). Defaults to the driving clip\'s length.'),
|
|
1799
|
+
videoResolution: z.enum(['480p', '720p', '1080p', '4k']).optional().describe('Seedance engine only. Default 1080p.'),
|
|
1800
|
+
aspectRatio: z.string().optional().describe('Seedance engine only. Default 16:9.'),
|
|
1801
|
+
seedanceFace: z.boolean().optional().describe('Seedance engine only — a character\'s face is in the clip/image (default TRUE for motion transfer). Bills the -face key.'),
|
|
1802
|
+
seedanceRealFace: z.boolean().optional().describe('Seedance engine only — the driving clip/subject shows a REAL person. Premium -realface key; REQUIRES realFaceConsent=true.'),
|
|
1803
|
+
realFaceConsent: z.boolean().optional().describe('MANDATORY with seedanceRealFace — set true only after the user explicitly confirms they hold rights/consent to the likeness.'),
|
|
1804
|
+
sourceVideoSeconds: z.number().optional().describe('Seedance engine: the driving clip\'s duration in seconds (from the asset listing, 2-15s). Feeds the vref cost key (input+output billing).'),
|
|
1508
1805
|
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1509
1806
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 confirm gate. Required — both tiers exceed.'),
|
|
1510
1807
|
}),
|
|
1511
1808
|
async run(input, ctx) {
|
|
1512
1809
|
const motionModel = input.motionModel ?? 'kling-mc-pro';
|
|
1513
|
-
const
|
|
1810
|
+
const isSeedance = motionModel === 'seedance-2';
|
|
1811
|
+
let costKey;
|
|
1812
|
+
let seedanceDuration = 0;
|
|
1813
|
+
if (isSeedance) {
|
|
1814
|
+
if (input.seedanceRealFace && !input.realFaceConsent) {
|
|
1815
|
+
return ok({
|
|
1816
|
+
requires_clarification: true,
|
|
1817
|
+
missing: ['realFaceConsent'],
|
|
1818
|
+
message: 'Real-person motion transfer needs consent: confirm with the user that they hold the rights/consent to this likeness, then retry with realFaceConsent=true.',
|
|
1819
|
+
});
|
|
1820
|
+
}
|
|
1821
|
+
if (!input.sourceVideoSeconds) {
|
|
1822
|
+
return ok({
|
|
1823
|
+
requires_clarification: true,
|
|
1824
|
+
missing: ['sourceVideoSeconds'],
|
|
1825
|
+
message: 'Seedance motion transfer bills combined input+output seconds. Pass sourceVideoSeconds (the driving clip\'s duration from slates_list_assets, must be 2-15s).',
|
|
1826
|
+
});
|
|
1827
|
+
}
|
|
1828
|
+
seedanceDuration = input.duration ?? Math.min(15, Math.max(4, Math.ceil(input.sourceVideoSeconds)));
|
|
1829
|
+
costKey = videoCostKey({
|
|
1830
|
+
model: 'seedance-2',
|
|
1831
|
+
duration: seedanceDuration,
|
|
1832
|
+
videoResolution: input.videoResolution ?? '1080p',
|
|
1833
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1834
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1835
|
+
videoRefSeconds: input.sourceVideoSeconds,
|
|
1836
|
+
});
|
|
1837
|
+
}
|
|
1838
|
+
else {
|
|
1839
|
+
costKey = motionModel === 'kling-mc-std' ? 'kling-mc-std-5s' : 'kling-mc-pro-5s';
|
|
1840
|
+
}
|
|
1514
1841
|
const cloud = ctx.cloud();
|
|
1515
1842
|
const registry = await cloud.get('/api/agent/models');
|
|
1516
1843
|
const entry = registry.models.find((m) => m.model === costKey);
|
|
@@ -1534,9 +1861,9 @@ export const generateMotionTransfer = {
|
|
|
1534
1861
|
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1535
1862
|
source_ref: source,
|
|
1536
1863
|
target_ref: target,
|
|
1537
|
-
message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${motionModel}. ` +
|
|
1864
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} for ${isSeedance ? `${seedanceDuration}s Seedance motion transfer` : `5s ${motionModel}`} (${costKey}). ` +
|
|
1538
1865
|
`Transferring motion from ${source} onto ${target}. ` +
|
|
1539
|
-
`Re-call with confirm=true after the user explicitly OKs the spend, or pick kling-mc-std to save $0.31. ` +
|
|
1866
|
+
`Re-call with confirm=true after the user explicitly OKs the spend${isSeedance ? '' : ', or pick kling-mc-std to save $0.31'}. ` +
|
|
1540
1867
|
`When discussing with the user, refer to the assets by those codes — they'll match the gallery badges.`,
|
|
1541
1868
|
});
|
|
1542
1869
|
}
|
|
@@ -1554,12 +1881,27 @@ export const generateMotionTransfer = {
|
|
|
1554
1881
|
klingProvider: input.klingProvider,
|
|
1555
1882
|
estimatedCost: totalCents,
|
|
1556
1883
|
background: input.background,
|
|
1884
|
+
// Seedance engine passthrough — the desktop delegates to the seedance
|
|
1885
|
+
// ref-to-video path (vref billing, face cascade, consent gate).
|
|
1886
|
+
...(isSeedance
|
|
1887
|
+
? {
|
|
1888
|
+
duration: seedanceDuration,
|
|
1889
|
+
videoResolution: input.videoResolution,
|
|
1890
|
+
aspectRatio: input.aspectRatio,
|
|
1891
|
+
seedanceFace: input.seedanceFace !== false && input.seedanceRealFace !== true,
|
|
1892
|
+
seedanceRealFace: input.seedanceRealFace === true,
|
|
1893
|
+
realFaceConsent: input.realFaceConsent === true,
|
|
1894
|
+
// Ride the caller-supplied clip duration through — the asset row's
|
|
1895
|
+
// duration can be null for imported clips.
|
|
1896
|
+
sourceVideoDurationSeconds: input.sourceVideoSeconds,
|
|
1897
|
+
}
|
|
1898
|
+
: {}),
|
|
1557
1899
|
});
|
|
1558
1900
|
if (!result.success)
|
|
1559
1901
|
throw new Error(result.error ?? 'Motion transfer generation failed');
|
|
1560
1902
|
if (result.background) {
|
|
1561
1903
|
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1562
|
-
return backgroundSubmitted(`5s motion transfer (${motionModel})`, ids, {
|
|
1904
|
+
return backgroundSubmitted(`${isSeedance ? `${seedanceDuration}s` : '5s'} motion transfer (${motionModel})`, ids, {
|
|
1563
1905
|
variant: costKey,
|
|
1564
1906
|
motionModel,
|
|
1565
1907
|
projectId: input.projectId,
|
|
@@ -1570,7 +1912,7 @@ export const generateMotionTransfer = {
|
|
|
1570
1912
|
});
|
|
1571
1913
|
}
|
|
1572
1914
|
return {
|
|
1573
|
-
text: `Generated 5s motion transfer (${motionModel}) into project ${input.projectId} ` +
|
|
1915
|
+
text: `Generated ${isSeedance ? `${seedanceDuration}s` : '5s'} motion transfer (${motionModel}) into project ${input.projectId} ` +
|
|
1574
1916
|
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
|
|
1575
1917
|
(input.prompt ? ` Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` : ''),
|
|
1576
1918
|
data: {
|
|
@@ -1587,6 +1929,132 @@ export const generateMotionTransfer = {
|
|
|
1587
1929
|
};
|
|
1588
1930
|
},
|
|
1589
1931
|
};
|
|
1932
|
+
// ── Edit video (Kling O3 video-to-video) ────────────────────────
|
|
1933
|
+
export const editVideo = {
|
|
1934
|
+
id: 'slates_edit_video',
|
|
1935
|
+
description: 'Edit an EXISTING video clip with one instruction via Kling O3 video-to-video edit — character swap, environment change, style transfer — in one pass, no masking. Original motion, camera, and audio are preserved; only what the prompt names changes. Use when a clip is ~90% right (fix it, don\'t re-roll it) or to AI-edit the user\'s own footage. Source clip constraints: 3–15s, 720–3840px, MP4/MOV. Cost = per second of OUTPUT (≈ clip length, rounded UP to the next second): kling-v3.0-omni-edit ≈ 19¢/s, kling-v3.0-omni-pro-edit ≈ 25¢/s. Subjects to swap IN go as characterAssetIds (frontal + angle images become Kling elements); style refs as styleAssetIds; max 4 combined. The edited clip saves as a NEW asset linked to its parent (chain edits freely). Routing: Kling edit is the default edit tool (element lock + audio intact); prefer Seedance edit/relocate only for style-transfer-heavy jobs — see slates-model-selection. Prompting: slates-prompting-kling-v3 §Edit.',
|
|
1936
|
+
input: z.object({
|
|
1937
|
+
projectId: z.string().uuid().describe('Project the source clip lives in.'),
|
|
1938
|
+
sourceVideoAssetId: z.string().describe('The VIDEO asset to edit — UUID or badge code ("VID-V3", bare "V3"); codes resolve against the project at call time. 3–15s clips only.'),
|
|
1939
|
+
prompt: z.string().min(1).max(2500).describe('The change, not the whole scene — e.g. "replace the man with @marcus", "make it a rainy night", "turn the street into a neon Tokyo alley". Mention subjects with @name; the transport compiles them to Kling\'s @ElementN notation.'),
|
|
1940
|
+
model: z.enum(['kling-v3.0-omni-edit', 'kling-v3.0-omni-pro-edit']).optional().describe('Default kling-v3.0-omni-edit. Pro (~25¢/s vs ~19¢/s) only for hero shots where fidelity matters.'),
|
|
1941
|
+
characterAssetIds: z.array(z.string()).max(4).optional().describe('Subject/element image assets to swap IN (UUIDs or badge codes). Each becomes a Kling element (@ElementN).'),
|
|
1942
|
+
styleAssetIds: z.array(z.string()).max(4).optional().describe('Style/appearance reference images (@ImageN). Max 4 combined with characterAssetIds.'),
|
|
1943
|
+
keepAudio: z.boolean().optional().describe('Preserve the original audio track (default true).'),
|
|
1944
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1945
|
+
confirm: z.boolean().optional().describe('Set true to bypass the cost confirm gate after the user OKs the spend.'),
|
|
1946
|
+
}),
|
|
1947
|
+
async run(input, ctx) {
|
|
1948
|
+
const desktop = ctx.desktop();
|
|
1949
|
+
await desktop.requireCapability('edit-video', 'video editing (Kling O3 edit)');
|
|
1950
|
+
if (input.background) {
|
|
1951
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1952
|
+
}
|
|
1953
|
+
const model = input.model ?? 'kling-v3.0-omni-edit';
|
|
1954
|
+
// Resolve refs (UUIDs or badge codes) against the project AT CALL TIME.
|
|
1955
|
+
const refInputs = [
|
|
1956
|
+
{ ref: input.sourceVideoAssetId, role: 'source clip' },
|
|
1957
|
+
...(input.characterAssetIds ?? []).map((ref) => ({ ref, role: 'subject element' })),
|
|
1958
|
+
...(input.styleAssetIds ?? []).map((ref) => ({ ref, role: 'style' })),
|
|
1959
|
+
];
|
|
1960
|
+
const resolved = await resolveAssetRefs(ctx, input.projectId, refInputs.map((r) => r.ref));
|
|
1961
|
+
const rid = (ref) => resolved.get(ref)?.id ?? ref;
|
|
1962
|
+
const sourceId = rid(input.sourceVideoAssetId);
|
|
1963
|
+
const characterAssetIds = (input.characterAssetIds ?? []).map(rid);
|
|
1964
|
+
const styleAssetIds = (input.styleAssetIds ?? []).map(rid);
|
|
1965
|
+
const refEcho = describeResolvedRefs(refInputs, resolved);
|
|
1966
|
+
if (characterAssetIds.length + styleAssetIds.length > 4) {
|
|
1967
|
+
throw new Error('Kling O3 edit takes max 4 combined subject + style references.');
|
|
1968
|
+
}
|
|
1969
|
+
// The billed key needs the clip's duration (ceiled). Read it from the
|
|
1970
|
+
// project's asset records — the desktop route independently re-validates.
|
|
1971
|
+
const { assets } = await desktop.get('/agent/assets', { projectId: input.projectId });
|
|
1972
|
+
const sourceRow = (assets ?? []).find((a) => String(a.id).toLowerCase() === sourceId.toLowerCase());
|
|
1973
|
+
if (!sourceRow)
|
|
1974
|
+
throw new Error(`Source asset not found in project: ${input.sourceVideoAssetId}`);
|
|
1975
|
+
if (sourceRow.type !== 'video')
|
|
1976
|
+
throw new Error('sourceVideoAssetId must reference a VIDEO asset.');
|
|
1977
|
+
const clipSeconds = Number(sourceRow.duration);
|
|
1978
|
+
if (!Number.isFinite(clipSeconds) || clipSeconds <= 0) {
|
|
1979
|
+
throw new Error('Source clip has no recorded duration — cannot quote the edit. Re-import the clip or pick another.');
|
|
1980
|
+
}
|
|
1981
|
+
if (clipSeconds > 15.05 || clipSeconds < 2.95) {
|
|
1982
|
+
throw new Error(`Source clip is ${clipSeconds.toFixed(1)}s — Kling O3 edit accepts 3–15s. Trim it first (agents can pre-trim on the timeline).`);
|
|
1983
|
+
}
|
|
1984
|
+
const billedSeconds = Math.min(15, Math.max(3, Math.ceil(clipSeconds - 0.05)));
|
|
1985
|
+
const costKey = klingEditCostKey(model, billedSeconds);
|
|
1986
|
+
const cloud = ctx.cloud();
|
|
1987
|
+
const registry = await cloud.get('/api/agent/models');
|
|
1988
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
1989
|
+
if (!entry)
|
|
1990
|
+
throw new Error(`Model variant not in registry: ${costKey}`);
|
|
1991
|
+
const totalCents = entry.cost_cents;
|
|
1992
|
+
// Confirm gate — look-first: preview the source clip + refs so the LLM
|
|
1993
|
+
// sees what it's editing before committing spend (mirrors generateVideo).
|
|
1994
|
+
if ((totalCents > 50 || refInputs.length > 1) && !input.confirm) {
|
|
1995
|
+
const previews = await previewAssets(ctx, [
|
|
1996
|
+
{ id: sourceId, type: 'video', role: 'source clip' },
|
|
1997
|
+
...characterAssetIds.map((id) => ({ id, type: 'image', role: 'subject element' })),
|
|
1998
|
+
...styleAssetIds.map((id) => ({ id, type: 'image', role: 'style' })),
|
|
1999
|
+
]);
|
|
2000
|
+
return {
|
|
2001
|
+
text: `Cost: $${(totalCents / 100).toFixed(2)} (${totalCents}¢) to edit a ${billedSeconds}s clip with ${model} (${costKey}). ` +
|
|
2002
|
+
`${refEcho} Re-call with confirm=true after the user explicitly OKs the spend.`,
|
|
2003
|
+
images: previews.flatMap((p) => p.images),
|
|
2004
|
+
data: {
|
|
2005
|
+
requires_confirm: true,
|
|
2006
|
+
model,
|
|
2007
|
+
variant: costKey,
|
|
2008
|
+
billed_seconds: billedSeconds,
|
|
2009
|
+
clip_seconds: clipSeconds,
|
|
2010
|
+
estimated_cents: totalCents,
|
|
2011
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
2012
|
+
references: previews.map((p) => ({ ref: p.ref, role: p.role, ...p.meta })),
|
|
2013
|
+
},
|
|
2014
|
+
};
|
|
2015
|
+
}
|
|
2016
|
+
const result = await desktop.post('/agent/generation/edit-video', {
|
|
2017
|
+
projectId: input.projectId,
|
|
2018
|
+
model,
|
|
2019
|
+
prompt: input.prompt,
|
|
2020
|
+
sourceVideoAssetId: sourceId,
|
|
2021
|
+
characterAssetIds,
|
|
2022
|
+
styleAssetIds,
|
|
2023
|
+
keepAudio: input.keepAudio !== false,
|
|
2024
|
+
background: input.background,
|
|
2025
|
+
});
|
|
2026
|
+
if (!result.success)
|
|
2027
|
+
throw new Error(result.error ?? 'Video edit failed');
|
|
2028
|
+
if (result.background) {
|
|
2029
|
+
const ids = result.generationId ? [result.generationId] : [];
|
|
2030
|
+
return backgroundSubmitted(`${billedSeconds}s video edit (${model})`, ids, {
|
|
2031
|
+
model,
|
|
2032
|
+
variant: costKey,
|
|
2033
|
+
projectId: input.projectId,
|
|
2034
|
+
sourceVideoAssetId: sourceId,
|
|
2035
|
+
cost_cents: totalCents,
|
|
2036
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
2037
|
+
}, refEcho);
|
|
2038
|
+
}
|
|
2039
|
+
return {
|
|
2040
|
+
text: `Edited ${billedSeconds}s clip saved as a new asset in project ${input.projectId} ` +
|
|
2041
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢) via ${model}. ` +
|
|
2042
|
+
`Edit: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` +
|
|
2043
|
+
(refEcho ? ` ${refEcho}` : ''),
|
|
2044
|
+
data: {
|
|
2045
|
+
model,
|
|
2046
|
+
variant: costKey,
|
|
2047
|
+
projectId: input.projectId,
|
|
2048
|
+
sourceVideoAssetId: sourceId,
|
|
2049
|
+
billed_seconds: billedSeconds,
|
|
2050
|
+
cost_cents: totalCents,
|
|
2051
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
2052
|
+
asset: result.asset,
|
|
2053
|
+
generationId: result.generationId,
|
|
2054
|
+
},
|
|
2055
|
+
};
|
|
2056
|
+
},
|
|
2057
|
+
};
|
|
1590
2058
|
// ── Generation status (background mode) ─────────────────────────
|
|
1591
2059
|
export const getGenerationStatus = {
|
|
1592
2060
|
id: 'slates_get_generation_status',
|
|
@@ -2085,6 +2553,8 @@ function resolveGuideTopic(topic) {
|
|
|
2085
2553
|
}
|
|
2086
2554
|
if (t.startsWith('nano-banana'))
|
|
2087
2555
|
return 'slates-prompting-nano-banana-2';
|
|
2556
|
+
if (t.startsWith('gpt-image') || t.startsWith('gpt image'))
|
|
2557
|
+
return 'slates-prompting-gpt-image-2';
|
|
2088
2558
|
if (t.startsWith('flux'))
|
|
2089
2559
|
return 'slates-prompting-flux-2-max';
|
|
2090
2560
|
if (t.startsWith('seedream'))
|
|
@@ -2093,6 +2563,8 @@ function resolveGuideTopic(topic) {
|
|
|
2093
2563
|
return 'slates-prompting-veo-3';
|
|
2094
2564
|
if (t.startsWith('kling-mc'))
|
|
2095
2565
|
return 'slates-prompting-motion-transfer';
|
|
2566
|
+
if (t === 'edit-video' || t === 'video-edit' || t === 'edit video' || t === 'video edit')
|
|
2567
|
+
return 'slates-prompting-kling-v3';
|
|
2096
2568
|
if (t.startsWith('kling-v3'))
|
|
2097
2569
|
return 'slates-prompting-kling-v3';
|
|
2098
2570
|
if (t.startsWith('seedance'))
|
|
@@ -2170,6 +2642,7 @@ export const ALL_OPERATIONS = [
|
|
|
2170
2642
|
generateVideo,
|
|
2171
2643
|
generateLipSync,
|
|
2172
2644
|
generateMotionTransfer,
|
|
2645
|
+
editVideo,
|
|
2173
2646
|
editImage,
|
|
2174
2647
|
getGenerationStatus,
|
|
2175
2648
|
listGenerations,
|