@slatesvideo/shared 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/clients/cloud.js +32 -1
- package/dist/clients/desktop.d.ts +10 -4
- package/dist/clients/desktop.js +95 -7
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/operations/index.d.ts +207 -1
- package/dist/operations/index.js +1513 -32
- package/dist/skills/content.d.ts +2 -0
- package/dist/skills/content.js +20 -0
- package/package.json +31 -4
- package/skills/slates-character-turnaround.md +53 -0
- package/skills/slates-cost-discipline.md +96 -0
- package/skills/slates-direct-response-ad.md +66 -0
- package/skills/slates-edit-and-iterate.md +48 -0
- package/skills/slates-one-prompt-film.md +67 -0
- package/skills/slates-prompting-flux-2-max.md +106 -0
- package/skills/slates-prompting-kling-v3.md +160 -0
- package/skills/slates-prompting-lip-sync.md +169 -0
- package/skills/slates-prompting-motion-transfer.md +148 -0
- package/skills/slates-prompting-nano-banana-2.md +147 -0
- package/skills/slates-prompting-seedance.md +144 -0
- package/skills/slates-prompting-seedream-5-lite.md +74 -0
- package/skills/slates-prompting-veo-3.md +149 -0
- package/skills/slates-storyboard-from-script.md +49 -0
- package/skills/slates-vision-feedback-loop.md +62 -0
- package/dist/auth.d.ts.map +0 -1
- package/dist/auth.js.map +0 -1
- package/dist/clients/cloud.d.ts.map +0 -1
- package/dist/clients/cloud.js.map +0 -1
- package/dist/clients/desktop.d.ts.map +0 -1
- package/dist/clients/desktop.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/operations/index.d.ts.map +0 -1
- package/dist/operations/index.js.map +0 -1
package/dist/operations/index.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { z } from 'zod';
|
|
13
13
|
import { SlatesCloudClient } from '../clients/cloud.js';
|
|
14
14
|
import { SlatesDesktopClient } from '../clients/desktop.js';
|
|
15
|
+
import { SKILLS } from '../skills/content.js';
|
|
15
16
|
export function defaultContext() {
|
|
16
17
|
return {
|
|
17
18
|
cloud: () => new SlatesCloudClient(),
|
|
@@ -25,6 +26,21 @@ function ok(data, text) {
|
|
|
25
26
|
data,
|
|
26
27
|
};
|
|
27
28
|
}
|
|
29
|
+
// Shared describe-text for the background flag on every generate_* op.
|
|
30
|
+
const BACKGROUND_DESCRIBE = 'Submit and return immediately with generationId(s) instead of blocking until the file is saved. ' +
|
|
31
|
+
'Poll with slates_get_generation_status. Recommended for video (1-5 min renders).';
|
|
32
|
+
// Early-return shape when a generation route accepted the job in background
|
|
33
|
+
// mode ({ background: true } in the response). No inline-image fetch — the
|
|
34
|
+
// asset doesn't exist yet; the poller delivers it on completion.
|
|
35
|
+
function backgroundSubmitted(kind, ids, extra) {
|
|
36
|
+
const idText = ids.length > 0 ? ids.join(', ') : '(no id returned)';
|
|
37
|
+
return {
|
|
38
|
+
text: `Submitted ${kind} in the background — generationId(s): ${idText}. ` +
|
|
39
|
+
`Poll slates_get_generation_status with each id every 5-15s until status is 'completed' ` +
|
|
40
|
+
`(video generations commonly take 1-5 minutes). Generations survive app restarts.`,
|
|
41
|
+
data: { generationIds: ids, status: 'processing', ...extra },
|
|
42
|
+
};
|
|
43
|
+
}
|
|
28
44
|
// ── Workspace + identity ────────────────────────────────────────
|
|
29
45
|
export const getWorkspaceState = {
|
|
30
46
|
id: 'slates_get_workspace_state',
|
|
@@ -135,17 +151,90 @@ export const listAssets = {
|
|
|
135
151
|
};
|
|
136
152
|
export const getAssetImage = {
|
|
137
153
|
id: 'slates_get_asset_image',
|
|
138
|
-
description: 'Read an asset image off disk and return it inline as base64
|
|
154
|
+
description: 'Read an asset image off disk and return it inline as base64 so the calling LLM sees the actual pixels. The response also carries the asset\'s short code (e.g. IMG-A12) and human label — USE these when referring to the asset in chat with the user (e.g. "I\'m using IMG-A12 — Beach Sunset as the first frame") so they can match it to the badged thumbnail in the Slates gallery. Default JPEG 85% ≤1024px long-edge for token economy. Pass fullRes=true for the original PNG.',
|
|
139
155
|
input: z.object({
|
|
140
156
|
id: z.string().uuid(),
|
|
141
157
|
fullRes: z.boolean().optional(),
|
|
142
158
|
}),
|
|
143
159
|
async run(input, ctx) {
|
|
144
160
|
const r = await ctx.desktop().get('/agent/assets/image', { id: input.id, fullRes: input.fullRes ?? false });
|
|
161
|
+
const refLabel = r.code ? (r.label ? `${r.code} — ${r.label}` : r.code) : r.asset_id;
|
|
145
162
|
return {
|
|
146
|
-
text: `Loaded
|
|
163
|
+
text: `Loaded ${refLabel} (${r.bytes} bytes, ${r.mimeType}). When referring to this asset to the user, use "${refLabel}" — they can find the matching badge in the Slates gallery.`,
|
|
147
164
|
images: [{ data: r.data, mimeType: r.mimeType }],
|
|
148
|
-
data: {
|
|
165
|
+
data: {
|
|
166
|
+
asset_id: r.asset_id,
|
|
167
|
+
code: r.code,
|
|
168
|
+
label: r.label,
|
|
169
|
+
file_path: r.file_path,
|
|
170
|
+
bytes: r.bytes,
|
|
171
|
+
mimeType: r.mimeType,
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
export const getAssetsBatch = {
|
|
177
|
+
id: 'slates_get_assets_batch',
|
|
178
|
+
description: 'Fetch up to 8 image-asset thumbnails inline in a single call. Use this when picking the right reference from a project gallery — one round trip beats N. Each returned image carries its short code (IMG-A12) and label so you can speak about candidates in the user\'s shared vocabulary ("between IMG-A12 and IMG-A14, the second has the right composition"). Video assets are not supported here — call slates_get_asset_video_frames for those.',
|
|
179
|
+
input: z.object({
|
|
180
|
+
ids: z.array(z.string().uuid()).min(1).max(8).describe('1-8 image-asset ids. Order is preserved in the response.'),
|
|
181
|
+
}),
|
|
182
|
+
async run(input, ctx) {
|
|
183
|
+
const r = await ctx.desktop().post('/agent/assets/batch-images', { ids: input.ids });
|
|
184
|
+
const refs = r.images.map((i) => i.code ? (i.label ? `${i.code} — ${i.label}` : i.code) : i.asset_id);
|
|
185
|
+
const errSummary = r.errors.length > 0
|
|
186
|
+
? ` Skipped ${r.errors.length}: ${r.errors.map((e) => `${e.asset_id} (${e.error})`).join('; ')}.`
|
|
187
|
+
: '';
|
|
188
|
+
return {
|
|
189
|
+
text: `Loaded ${r.images.length} image(s): ${refs.join(', ')}.` +
|
|
190
|
+
errSummary +
|
|
191
|
+
` When discussing these with the user, reference them by their codes so they can match each one to a gallery badge.`,
|
|
192
|
+
images: r.images.map((i) => ({ data: i.data, mimeType: i.mimeType })),
|
|
193
|
+
data: {
|
|
194
|
+
images: r.images.map((i) => ({
|
|
195
|
+
asset_id: i.asset_id,
|
|
196
|
+
code: i.code,
|
|
197
|
+
label: i.label,
|
|
198
|
+
file_path: i.file_path,
|
|
199
|
+
bytes: i.bytes,
|
|
200
|
+
mimeType: i.mimeType,
|
|
201
|
+
})),
|
|
202
|
+
errors: r.errors,
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
},
|
|
206
|
+
};
|
|
207
|
+
export const getAssetVideoFrames = {
|
|
208
|
+
id: 'slates_get_asset_video_frames',
|
|
209
|
+
description: 'Extract N evenly-spaced keyframes from a video asset and return them inline as base64 JPEGs. This is the "see the video" path — LLMs can\'t consume video natively, so frames are the next best thing. Default 3 frames (start / middle / end). Bump to 5-8 for longer clips or when motion is the whole story. Use this before writing a motion-transfer prompt, a lip-sync refinement, or any iteration on a video clip. Response carries the asset\'s code (e.g. VID-V3) + label — name them when discussing the clip with the user.',
|
|
210
|
+
input: z.object({
|
|
211
|
+
id: z.string().uuid(),
|
|
212
|
+
count: z.number().int().min(1).max(8).optional().describe('Number of frames to extract. Default 3.'),
|
|
213
|
+
}),
|
|
214
|
+
async run(input, ctx) {
|
|
215
|
+
const params = { id: input.id };
|
|
216
|
+
if (input.count != null)
|
|
217
|
+
params.count = input.count;
|
|
218
|
+
const r = await ctx.desktop().get('/agent/assets/video-frames', params);
|
|
219
|
+
const refLabel = r.code ? (r.label ? `${r.code} — ${r.label}` : r.code) : r.asset_id;
|
|
220
|
+
const timestamps = r.frames.map((f) => `${f.timestamp_seconds}s`).join(', ');
|
|
221
|
+
return {
|
|
222
|
+
text: `Extracted ${r.frames.length} frame(s) from ${refLabel} ` +
|
|
223
|
+
`(${r.duration_seconds}s clip) at: ${timestamps}. ` +
|
|
224
|
+
`When referring to this video with the user, call it "${refLabel}".`,
|
|
225
|
+
images: r.frames.map((f) => ({ data: f.data, mimeType: f.mimeType })),
|
|
226
|
+
data: {
|
|
227
|
+
asset_id: r.asset_id,
|
|
228
|
+
code: r.code,
|
|
229
|
+
label: r.label,
|
|
230
|
+
file_path: r.file_path,
|
|
231
|
+
duration_seconds: r.duration_seconds,
|
|
232
|
+
frames: r.frames.map((f) => ({
|
|
233
|
+
timestamp_seconds: f.timestamp_seconds,
|
|
234
|
+
bytes: f.bytes,
|
|
235
|
+
mimeType: f.mimeType,
|
|
236
|
+
})),
|
|
237
|
+
},
|
|
149
238
|
};
|
|
150
239
|
},
|
|
151
240
|
};
|
|
@@ -345,61 +434,334 @@ export const addFrame = {
|
|
|
345
434
|
return ok(await ctx.desktop().post('/agent/frames', input));
|
|
346
435
|
},
|
|
347
436
|
};
|
|
437
|
+
function formatRef(asset) {
|
|
438
|
+
if (!asset.code)
|
|
439
|
+
return asset.asset_id;
|
|
440
|
+
return asset.label ? `${asset.code} — ${asset.label}` : asset.code;
|
|
441
|
+
}
|
|
442
|
+
// Cheap metadata-only lookup used by the mechanical-op confirm gates
|
|
443
|
+
// (motion_transfer, lip_sync). Returns the user-facing reference string
|
|
444
|
+
// without pulling any pixels — purely so the agent can announce the
|
|
445
|
+
// chosen assets to the user in shared vocabulary.
|
|
446
|
+
async function lookupAssetRef(desktop, id) {
|
|
447
|
+
try {
|
|
448
|
+
const r = await desktop.get('/agent/assets/path', { id });
|
|
449
|
+
return formatRef({ asset_id: id, code: r.code ?? null, label: r.label ?? null });
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
return id;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
async function previewAsset(ctx, id, type, role, videoFrameCount = 3) {
|
|
456
|
+
const desktop = ctx.desktop();
|
|
457
|
+
if (type === 'image') {
|
|
458
|
+
const r = await desktop.get('/agent/assets/image', { id });
|
|
459
|
+
return {
|
|
460
|
+
ref: formatRef(r),
|
|
461
|
+
role,
|
|
462
|
+
images: [{ data: r.data, mimeType: r.mimeType }],
|
|
463
|
+
meta: {
|
|
464
|
+
asset_id: r.asset_id,
|
|
465
|
+
code: r.code,
|
|
466
|
+
label: r.label,
|
|
467
|
+
type: 'image',
|
|
468
|
+
file_path: r.file_path,
|
|
469
|
+
},
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
const r = await desktop.get('/agent/assets/video-frames', { id, count: videoFrameCount });
|
|
473
|
+
return {
|
|
474
|
+
ref: formatRef(r),
|
|
475
|
+
role,
|
|
476
|
+
images: r.frames.map((f) => ({ data: f.data, mimeType: f.mimeType })),
|
|
477
|
+
meta: {
|
|
478
|
+
asset_id: r.asset_id,
|
|
479
|
+
code: r.code,
|
|
480
|
+
label: r.label,
|
|
481
|
+
type: 'video',
|
|
482
|
+
file_path: r.file_path,
|
|
483
|
+
frame_count: r.frames.length,
|
|
484
|
+
},
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
async function previewAssets(ctx, refs) {
|
|
488
|
+
// Sequential — desktop is local, parallel buys nothing and the order
|
|
489
|
+
// matters for the confirm-gate text (first frame, last frame, ingredients...).
|
|
490
|
+
const out = [];
|
|
491
|
+
for (const r of refs) {
|
|
492
|
+
try {
|
|
493
|
+
out.push(await previewAsset(ctx, r.id, r.type, r.role));
|
|
494
|
+
}
|
|
495
|
+
catch (err) {
|
|
496
|
+
// Best-effort previews — a missing frame shouldn't block the gen,
|
|
497
|
+
// but flag it so the agent sees the gap.
|
|
498
|
+
out.push({
|
|
499
|
+
ref: `[unavailable] ${r.id}`,
|
|
500
|
+
role: r.role,
|
|
501
|
+
images: [],
|
|
502
|
+
meta: {
|
|
503
|
+
asset_id: r.id,
|
|
504
|
+
code: null,
|
|
505
|
+
label: null,
|
|
506
|
+
type: r.type,
|
|
507
|
+
file_path: '',
|
|
508
|
+
},
|
|
509
|
+
});
|
|
510
|
+
void err;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
return out;
|
|
514
|
+
}
|
|
348
515
|
// ── Generation (cloud-routed, credits-default) ──────────────────
|
|
516
|
+
// Registry cost-key for an image model+resolution. Mirrors imageCreditKey()
|
|
517
|
+
// in slate/src/shared/pricing.ts: NB2 prices per resolution, FLUX.2 Max prices
|
|
518
|
+
// per resolution (1k is the bare key), Seedream is flat (one key).
|
|
519
|
+
function imageCostKey(model, resolution) {
|
|
520
|
+
if (model === 'flux-2-max')
|
|
521
|
+
return resolution === '1k' ? 'flux-2-max' : `flux-2-max-${resolution}`;
|
|
522
|
+
if (model === 'seedream-5-lite')
|
|
523
|
+
return 'seedream-5-lite';
|
|
524
|
+
return `nano-banana-2-${resolution}`;
|
|
525
|
+
}
|
|
349
526
|
export const generateImage = {
|
|
350
527
|
id: 'slates_generate_image',
|
|
351
|
-
description: 'Generate an image via Slates credits
|
|
528
|
+
description: 'Generate an image via Slates credits. Three models: nano-banana-2 (Google Gemini 3 Image — default, strongest general image model, well-censored), flux-2-max (FLUX.2 Max — photoreal, less censored, up to 4MP), seedream-5-lite (cheapest at ~$0.05 flat, less censored). Pass projectId to save into a Slates project (recommended — asset appears live in the desktop UI). FLUX.2 Max and Seedream 5 Lite REQUIRE projectId (no headless path). REQUIRED before calling: read the slates-cost-discipline skill (and slates-prompting-nano-banana-2 when using nano-banana-2). You MUST pass aspectRatio and resolution explicitly (the server returns requires_clarification when missing — defaults waste credits). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. MCP/CLI generation always charges credits. No skill files installed? Call slates_get_prompting_guide with the model\'s topic (and \'slates-cost-discipline\') before first use.',
|
|
352
529
|
input: z.object({
|
|
353
530
|
prompt: z.string().min(1).max(4000),
|
|
354
|
-
|
|
355
|
-
|
|
531
|
+
model: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional().describe('Image model. Default nano-banana-2. Use flux-2-max for photoreal / less-censored, seedream-5-lite for cheapest. flux-2-max & seedream-5-lite require projectId.'),
|
|
532
|
+
projectId: z.string().uuid().optional().describe('Save into this Slates project. Renderer refreshes live. Required for flux-2-max / seedream-5-lite.'),
|
|
533
|
+
resolution: z.enum(['1k', '2k', '4k']).optional().describe('Pick deliberately: 1k drafts, 2k hero shots, 4k print/final. (Seedream is flat-priced regardless; FLUX & NB2 price by resolution.) Never default this.'),
|
|
534
|
+
aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Pick deliberately from the use case. Cinematic → 16:9. TikTok/Reels/Story → 9:16. IG square → 1:1. Ultra-wide → 21:9. Ask the user when ambiguous.'),
|
|
356
535
|
count: z.number().int().min(1).max(4).optional(),
|
|
357
|
-
referenceImageUrls: z.array(z.string().url()).max(14).optional(),
|
|
536
|
+
referenceImageUrls: z.array(z.string().url()).max(14).optional().describe('Headless (no-projectId) nano-banana-2 only: up to 14 ref URLs. For projectId runs, upload refs with slates_upload_reference_image first. Always label each image\'s role in the prompt text.'),
|
|
537
|
+
referenceAssetIds: z.array(z.string().uuid()).max(14).optional().describe("Project asset ids to use as reference/ingredient images (resolved on the desktop). Requires projectId. For nano-banana-2 up to 14 refs; FLUX/Seedream route to their edit endpoints with lower per-model caps. Label each reference's role in the prompt text."),
|
|
538
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
358
539
|
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
|
|
359
540
|
}),
|
|
360
541
|
async run(input, ctx) {
|
|
361
|
-
|
|
362
|
-
|
|
542
|
+
// Clarification gate: aspectRatio + resolution must be deliberate.
|
|
543
|
+
// Mirrors the cost confirm gate — defaults silently wasted credits
|
|
544
|
+
// (1:1 when user wanted 16:9, 4k when 1k would have done). The LLM
|
|
545
|
+
// is forced to ask the user or read the skill instead of guessing.
|
|
546
|
+
if (!input.aspectRatio || !input.resolution) {
|
|
547
|
+
const missing = [];
|
|
548
|
+
if (!input.aspectRatio)
|
|
549
|
+
missing.push('aspectRatio');
|
|
550
|
+
if (!input.resolution)
|
|
551
|
+
missing.push('resolution');
|
|
552
|
+
return ok({
|
|
553
|
+
requires_clarification: true,
|
|
554
|
+
missing,
|
|
555
|
+
message: `Missing required field(s): ${missing.join(', ')}. ` +
|
|
556
|
+
`Read the slates-prompting-nano-banana-2 + slates-cost-discipline skills, ` +
|
|
557
|
+
`or ask the user. Aspect ratio options: 1:1 16:9 9:16 4:3 3:4 21:9 9:21 4:5 5:4 2:3 3:2. ` +
|
|
558
|
+
`Resolution options: 1k 2k 4k (same price band — pick by need, not cost).`,
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
const resolution = input.resolution;
|
|
562
|
+
const imageModel = input.model ?? 'nano-banana-2';
|
|
563
|
+
// FLUX.2 Max / Seedream 5 Lite have no headless path — they route through
|
|
564
|
+
// the desktop generation pipeline, which needs a project.
|
|
565
|
+
if (imageModel !== 'nano-banana-2' && !input.projectId) {
|
|
566
|
+
return ok({
|
|
567
|
+
requires_clarification: true,
|
|
568
|
+
missing: ['projectId'],
|
|
569
|
+
message: `${imageModel} requires projectId (no headless path). Use slates_list_projects / slates_create_project, then re-call with projectId.`,
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
// referenceAssetIds are PROJECT assets — the desktop resolves them off
|
|
573
|
+
// disk, so a headless run has nowhere to look them up. Same for
|
|
574
|
+
// background mode: the poller (slates_get_generation_status) reads the
|
|
575
|
+
// desktop's generation records.
|
|
576
|
+
const referenceAssetIds = input.referenceAssetIds ?? [];
|
|
577
|
+
if (referenceAssetIds.length > 0 && !input.projectId) {
|
|
578
|
+
return ok({
|
|
579
|
+
requires_clarification: true,
|
|
580
|
+
missing: ['projectId'],
|
|
581
|
+
message: 'referenceAssetIds are project assets resolved on the desktop — pass the projectId they live in (slates_list_projects to find it).',
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
if (input.background && !input.projectId) {
|
|
585
|
+
return ok({
|
|
586
|
+
requires_clarification: true,
|
|
587
|
+
missing: ['projectId'],
|
|
588
|
+
message: 'background=true routes through the desktop generation pipeline (so slates_get_generation_status can poll it) — pass a projectId, or drop background for a blocking headless run.',
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
if (referenceAssetIds.length > 0) {
|
|
592
|
+
await ctx.desktop().requireCapability('image-references', 'reference images on image generation');
|
|
593
|
+
}
|
|
594
|
+
if (input.background) {
|
|
595
|
+
await ctx.desktop().requireCapability('background-generation', 'background generation');
|
|
596
|
+
}
|
|
597
|
+
const costKey = imageCostKey(imageModel, resolution);
|
|
363
598
|
const cloud = ctx.cloud();
|
|
364
599
|
const registry = await cloud.get('/api/agent/models');
|
|
365
|
-
const entry = registry.models.find((m) => m.model ===
|
|
600
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
366
601
|
if (!entry)
|
|
367
|
-
throw new Error(`Model not in registry: ${
|
|
602
|
+
throw new Error(`Model not in registry: ${costKey}`);
|
|
368
603
|
const totalCents = entry.cost_cents * (input.count ?? 1);
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
604
|
+
// Confirm gate. Fires on cost > $0.50, AND (look-first, mirroring
|
|
605
|
+
// slates_generate_video) whenever reference assets are involved
|
|
606
|
+
// regardless of cost — the LLM must see what it's referencing before
|
|
607
|
+
// committing spend.
|
|
608
|
+
if ((totalCents > 50 || referenceAssetIds.length > 0) && !input.confirm) {
|
|
609
|
+
if (referenceAssetIds.length === 0) {
|
|
610
|
+
return ok({
|
|
611
|
+
requires_confirm: true,
|
|
612
|
+
model: costKey,
|
|
613
|
+
estimated_cents: totalCents,
|
|
614
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
615
|
+
message: 'Cost exceeds $0.50. Re-call with confirm=true to proceed, or pick a smaller resolution / count.',
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
const previews = await previewAssets(ctx, referenceAssetIds.map((id) => ({ id, type: 'image', role: 'reference' })));
|
|
619
|
+
const refLines = previews.map((p) => ` - ${p.role}: ${p.ref}`).join('\n');
|
|
620
|
+
return {
|
|
621
|
+
text: `Pre-flight for ${imageModel} (${costKey}): ` +
|
|
622
|
+
`$${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
|
|
623
|
+
`\n\nReference images attached above:\n${refLines}\n\n` +
|
|
624
|
+
`Review them against your prompt — every reference's role must be labeled in the prompt text. ` +
|
|
625
|
+
`If the references suggest a different composition / style than the current prompt captures, REVISE the prompt before confirming. ` +
|
|
626
|
+
`When you talk to the user about this gen, refer to each reference by its code (e.g. "${previews[0]?.ref ?? 'IMG-A?'}") — they'll see the matching badge in the Slates gallery.` +
|
|
627
|
+
`\n\nWhen ready, re-call slates_generate_image with confirm=true and the (possibly revised) prompt.`,
|
|
628
|
+
images: previews.flatMap((p) => p.images),
|
|
629
|
+
data: {
|
|
630
|
+
requires_confirm: true,
|
|
631
|
+
model: imageModel,
|
|
632
|
+
variant: costKey,
|
|
633
|
+
estimated_cents: totalCents,
|
|
634
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
635
|
+
references: previews.map((p) => ({
|
|
636
|
+
role: p.role,
|
|
637
|
+
ref: p.ref,
|
|
638
|
+
asset_id: p.meta.asset_id,
|
|
639
|
+
code: p.meta.code,
|
|
640
|
+
label: p.meta.label,
|
|
641
|
+
type: p.meta.type,
|
|
642
|
+
})),
|
|
643
|
+
},
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
// Two paths:
|
|
647
|
+
// 1. projectId given → route through the desktop's normal generation
|
|
648
|
+
// pipeline. Eric sees the progress card light up, the asset save
|
|
649
|
+
// lands in the project, the gallery refreshes, and every
|
|
650
|
+
// renderer-side hook fires identically to a UI-triggered run.
|
|
651
|
+
// 2. no projectId → run headless via the cloud proxy and return
|
|
652
|
+
// bytes inline only (useful for headless agents that don't have
|
|
653
|
+
// a desktop attached).
|
|
654
|
+
if (input.projectId) {
|
|
655
|
+
const desktop = ctx.desktop();
|
|
656
|
+
const result = await desktop.post('/agent/generation/image', {
|
|
657
|
+
projectId: input.projectId,
|
|
658
|
+
prompt: input.prompt,
|
|
659
|
+
model: imageModel,
|
|
660
|
+
resolution,
|
|
661
|
+
aspectRatio: input.aspectRatio ?? '1:1',
|
|
662
|
+
count: input.count ?? 1,
|
|
663
|
+
...(referenceAssetIds.length > 0 ? { referenceAssetIds } : {}),
|
|
664
|
+
background: input.background,
|
|
376
665
|
});
|
|
666
|
+
// Partial multi-image failure: the desktop attaches an error message
|
|
667
|
+
// but result.assets still holds the images that DID save. Throwing
|
|
668
|
+
// here would discard real, paid-for assets and invite a full retry
|
|
669
|
+
// that double-spends — surface what landed instead.
|
|
670
|
+
const partialFailure = !result.success && Array.isArray(result.assets) && result.assets.length > 0;
|
|
671
|
+
if (!result.success && !partialFailure) {
|
|
672
|
+
throw new Error(result.error ?? 'Generation failed');
|
|
673
|
+
}
|
|
674
|
+
if (result.background) {
|
|
675
|
+
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
676
|
+
return backgroundSubmitted(`${imageModel} image generation`, ids, {
|
|
677
|
+
model: imageModel,
|
|
678
|
+
costKey,
|
|
679
|
+
projectId: input.projectId,
|
|
680
|
+
cost_cents: totalCents,
|
|
681
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
const assetList = result.assets
|
|
685
|
+
?? (result.asset ? [result.asset] : []);
|
|
686
|
+
const images = [];
|
|
687
|
+
for (const asset of assetList) {
|
|
688
|
+
const id = asset.id;
|
|
689
|
+
if (!id)
|
|
690
|
+
continue;
|
|
691
|
+
try {
|
|
692
|
+
const img = await desktop.get('/agent/assets/image', { id });
|
|
693
|
+
images.push({ data: img.data, mimeType: img.mimeType });
|
|
694
|
+
}
|
|
695
|
+
catch {
|
|
696
|
+
// Vision payload is best-effort; skip if the disk read fails.
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
const requestedCount = input.count ?? 1;
|
|
700
|
+
return {
|
|
701
|
+
text: partialFailure
|
|
702
|
+
? `Partial result: ${assetList.length} of ${requestedCount} image(s) saved into project ${input.projectId} ` +
|
|
703
|
+
`(error on the rest: ${result.error ?? 'unknown error'}). ` +
|
|
704
|
+
`The saved assets are in data.assets — re-generate only the missing count, don't redo the whole batch. ` +
|
|
705
|
+
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`
|
|
706
|
+
: `Generated ${assetList.length} image(s) into project ${input.projectId} ` +
|
|
707
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
708
|
+
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
|
|
709
|
+
images,
|
|
710
|
+
data: {
|
|
711
|
+
model: imageModel,
|
|
712
|
+
costKey,
|
|
713
|
+
projectId: input.projectId,
|
|
714
|
+
aspectRatio: input.aspectRatio,
|
|
715
|
+
resolution,
|
|
716
|
+
cost_cents: totalCents,
|
|
717
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
718
|
+
assets: assetList,
|
|
719
|
+
generationIds: result.generationIds ?? (result.generationId ? [result.generationId] : []),
|
|
720
|
+
...(partialFailure
|
|
721
|
+
? { partial: true, error: result.error ?? 'unknown error', requested_count: requestedCount }
|
|
722
|
+
: {}),
|
|
723
|
+
},
|
|
724
|
+
};
|
|
377
725
|
}
|
|
378
|
-
//
|
|
379
|
-
//
|
|
380
|
-
//
|
|
381
|
-
//
|
|
382
|
-
//
|
|
383
|
-
//
|
|
384
|
-
//
|
|
726
|
+
// /proxy/generate kicks off a credit-aware job and returns a jobId
|
|
727
|
+
// for fal/Veo (async providers). We poll /proxy/jobs/{jobId} until
|
|
728
|
+
// the status is `completed` or `failed`, then fetch each image URL
|
|
729
|
+
// and inline as base64 so the calling LLM sees the pixels.
|
|
730
|
+
// The fal endpoint differs by mode: bare model id for text-to-image,
|
|
731
|
+
// `/edit` when reference image URLs are provided. Sending text-to-image
|
|
732
|
+
// to `/edit` 422s on the missing `image_urls` field.
|
|
733
|
+
const hasReferenceImages = !!input.referenceImageUrls && input.referenceImageUrls.length > 0;
|
|
734
|
+
const endpoint = hasReferenceImages
|
|
735
|
+
? 'fal-ai/nano-banana-2/edit'
|
|
736
|
+
: 'fal-ai/nano-banana-2';
|
|
385
737
|
const proxyResult = await cloud.post('/proxy/generate', {
|
|
386
738
|
provider: 'fal',
|
|
387
739
|
type: 'image',
|
|
388
|
-
model,
|
|
389
|
-
endpoint
|
|
740
|
+
model: costKey,
|
|
741
|
+
endpoint,
|
|
390
742
|
params: {
|
|
391
743
|
prompt: input.prompt,
|
|
392
744
|
aspect_ratio: input.aspectRatio ?? '1:1',
|
|
393
745
|
num_images: input.count ?? 1,
|
|
394
|
-
...(
|
|
746
|
+
...(hasReferenceImages
|
|
395
747
|
? { image_urls: input.referenceImageUrls }
|
|
396
748
|
: {}),
|
|
397
749
|
},
|
|
398
750
|
});
|
|
399
|
-
if (!proxyResult.success
|
|
751
|
+
if (!proxyResult.success) {
|
|
400
752
|
throw new Error(proxyResult.error ?? 'Generation failed');
|
|
401
753
|
}
|
|
402
|
-
|
|
754
|
+
let imagesPayload = proxyResult.data?.images;
|
|
755
|
+
if (!imagesPayload && proxyResult.jobId) {
|
|
756
|
+
const completed = await pollProxyJob(cloud, proxyResult.jobId);
|
|
757
|
+
imagesPayload = completed.images;
|
|
758
|
+
}
|
|
759
|
+
const urls = (imagesPayload ?? [])
|
|
760
|
+
.map((i) => i.url)
|
|
761
|
+
.filter((u) => !!u);
|
|
762
|
+
if (urls.length === 0) {
|
|
763
|
+
throw new Error('Generation completed but returned no image URLs.');
|
|
764
|
+
}
|
|
403
765
|
const images = [];
|
|
404
766
|
for (const url of urls) {
|
|
405
767
|
const r = await fetch(url);
|
|
@@ -408,9 +770,1090 @@ export const generateImage = {
|
|
|
408
770
|
images.push({ data: buf.toString('base64'), mimeType: mt });
|
|
409
771
|
}
|
|
410
772
|
return {
|
|
411
|
-
text: `Generated ${urls.length} image(s) for
|
|
773
|
+
text: `Generated ${urls.length} image(s) for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
774
|
+
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
|
|
775
|
+
images,
|
|
776
|
+
data: {
|
|
777
|
+
urls,
|
|
778
|
+
model: imageModel,
|
|
779
|
+
costKey,
|
|
780
|
+
aspectRatio: input.aspectRatio,
|
|
781
|
+
resolution,
|
|
782
|
+
cost_cents: totalCents,
|
|
783
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
784
|
+
},
|
|
785
|
+
};
|
|
786
|
+
},
|
|
787
|
+
};
|
|
788
|
+
async function pollProxyJob(cloud, jobId, options = {}) {
|
|
789
|
+
const intervalMs = options.intervalMs ?? 3000;
|
|
790
|
+
const timeoutMs = options.timeoutMs ?? 5 * 60 * 1000;
|
|
791
|
+
const start = Date.now();
|
|
792
|
+
while (Date.now() - start < timeoutMs) {
|
|
793
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
794
|
+
let status;
|
|
795
|
+
try {
|
|
796
|
+
status = await cloud.get(`/proxy/jobs/${jobId}`);
|
|
797
|
+
}
|
|
798
|
+
catch {
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
801
|
+
if (status.status === 'completed')
|
|
802
|
+
return status.data ?? {};
|
|
803
|
+
if (status.status === 'failed') {
|
|
804
|
+
throw new Error(status.error ?? 'Generation failed');
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
throw new Error(`Generation timed out after ${Math.round(timeoutMs / 1000)}s.`);
|
|
808
|
+
}
|
|
809
|
+
// ── Edit image ──────────────────────────────────────────────────
|
|
810
|
+
export const editImage = {
|
|
811
|
+
id: 'slates_edit_image',
|
|
812
|
+
description: 'Surgically edit an existing image asset with a text instruction (e.g. \'remove the lamppost\', \'make the jacket red\') instead of regenerating from scratch — use when ~90% of the image is already right. The edited result is saved as a NEW asset in the project (prompt prefixed \'[Edit]\'); the source is untouched. Default model nano-banana-2 (only model that also accepts referenceAssetIds); flux-2-max / seedream-5-lite use their own edit endpoints and ignore references. Before first use call slates_get_prompting_guide with topic \'slates-edit-and-iterate\'.',
|
|
813
|
+
input: z.object({
|
|
814
|
+
projectId: z.string().uuid(),
|
|
815
|
+
sourceAssetId: z.string().uuid().describe('Image asset to edit. Must already exist in the project.'),
|
|
816
|
+
prompt: z.string().min(1).max(4000).describe('The edit instruction — describe the change, not the whole image.'),
|
|
817
|
+
editModel: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional(),
|
|
818
|
+
referenceAssetIds: z.array(z.string().uuid()).max(13).optional().describe('nano-banana-2 only: extra reference images.'),
|
|
819
|
+
resolution: z.enum(['1k', '2k', '4k']).optional(),
|
|
820
|
+
aspectRatio: z.string().optional(),
|
|
821
|
+
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
|
|
822
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
823
|
+
}),
|
|
824
|
+
async run(input, ctx) {
|
|
825
|
+
const desktop = ctx.desktop();
|
|
826
|
+
await desktop.requireCapability('edit-image', 'image editing');
|
|
827
|
+
if (input.background) {
|
|
828
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
829
|
+
}
|
|
830
|
+
const editModel = input.editModel ?? 'nano-banana-2';
|
|
831
|
+
const resolution = input.resolution ?? '2k';
|
|
832
|
+
// NB2 edits charge the normal per-resolution NB2 key; FLUX / Seedream
|
|
833
|
+
// route to their dedicated edit endpoints, priced under '-edit' keys.
|
|
834
|
+
const costKey = editModel === 'nano-banana-2'
|
|
835
|
+
? imageCostKey('nano-banana-2', resolution)
|
|
836
|
+
: `${imageCostKey(editModel, resolution)}-edit`;
|
|
837
|
+
const cloud = ctx.cloud();
|
|
838
|
+
const registry = await cloud.get('/api/agent/models');
|
|
839
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
840
|
+
if (!entry)
|
|
841
|
+
throw new Error(`Model not in registry: ${costKey}`);
|
|
842
|
+
const totalCents = entry.cost_cents;
|
|
843
|
+
if (totalCents > 50 && !input.confirm) {
|
|
844
|
+
const sourceRef = await lookupAssetRef(desktop, input.sourceAssetId);
|
|
845
|
+
return ok({
|
|
846
|
+
requires_confirm: true,
|
|
847
|
+
variant: costKey,
|
|
848
|
+
estimated_cents: totalCents,
|
|
849
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
850
|
+
source_ref: sourceRef,
|
|
851
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} to edit ${sourceRef} with ${editModel} (${costKey}). ` +
|
|
852
|
+
`Re-call with confirm=true after the user explicitly OKs the spend. ` +
|
|
853
|
+
`When discussing with the user, refer to the source by its code (matches the gallery badge).`,
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
const result = await desktop.post('/agent/generation/edit-image', {
|
|
857
|
+
projectId: input.projectId,
|
|
858
|
+
sourceAssetId: input.sourceAssetId,
|
|
859
|
+
prompt: input.prompt,
|
|
860
|
+
editModel,
|
|
861
|
+
referenceAssetIds: input.referenceAssetIds,
|
|
862
|
+
resolution,
|
|
863
|
+
aspectRatio: input.aspectRatio,
|
|
864
|
+
background: input.background,
|
|
865
|
+
});
|
|
866
|
+
if (!result.success)
|
|
867
|
+
throw new Error(result.error ?? 'Image edit failed');
|
|
868
|
+
if (result.background) {
|
|
869
|
+
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
870
|
+
return backgroundSubmitted(`${editModel} image edit`, ids, {
|
|
871
|
+
editModel,
|
|
872
|
+
variant: costKey,
|
|
873
|
+
projectId: input.projectId,
|
|
874
|
+
sourceAssetId: input.sourceAssetId,
|
|
875
|
+
cost_cents: totalCents,
|
|
876
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
// Inline the edited result so the LLM sees whether the surgery landed —
|
|
880
|
+
// same best-effort pattern as slates_generate_image.
|
|
881
|
+
const images = [];
|
|
882
|
+
const newAssetId = result.asset?.id;
|
|
883
|
+
if (newAssetId) {
|
|
884
|
+
try {
|
|
885
|
+
const img = await desktop.get('/agent/assets/image', { id: newAssetId });
|
|
886
|
+
images.push({ data: img.data, mimeType: img.mimeType });
|
|
887
|
+
}
|
|
888
|
+
catch {
|
|
889
|
+
// Vision payload is best-effort; skip if the disk read fails.
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
return {
|
|
893
|
+
text: `Edited image saved as a new asset in project ${input.projectId} ` +
|
|
894
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢) via ${editModel}. ` +
|
|
895
|
+
`Edit: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
|
|
412
896
|
images,
|
|
413
|
-
data: {
|
|
897
|
+
data: {
|
|
898
|
+
editModel,
|
|
899
|
+
variant: costKey,
|
|
900
|
+
projectId: input.projectId,
|
|
901
|
+
sourceAssetId: input.sourceAssetId,
|
|
902
|
+
resolution,
|
|
903
|
+
cost_cents: totalCents,
|
|
904
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
905
|
+
asset: result.asset,
|
|
906
|
+
generationId: result.generationId,
|
|
907
|
+
},
|
|
908
|
+
};
|
|
909
|
+
},
|
|
910
|
+
};
|
|
911
|
+
// ── Generate video ──────────────────────────────────────────────
|
|
912
|
+
const VIDEO_MODELS = [
|
|
913
|
+
'kling-v3.0-std',
|
|
914
|
+
'kling-v3.0-pro',
|
|
915
|
+
'kling-v3.0-omni',
|
|
916
|
+
'veo-3.1-fast',
|
|
917
|
+
'veo-3.1-standard',
|
|
918
|
+
'seedance-2-fast',
|
|
919
|
+
'seedance-2-std',
|
|
920
|
+
];
|
|
921
|
+
// Model → registry cost-key. Each provider's keys ship with their own
|
|
922
|
+
// shape (verified against /api/agent/models):
|
|
923
|
+
// Kling: kling-v3-{standard|pro|omni}-{N}s — note the user-facing
|
|
924
|
+
// model id `kling-v3.0-std` maps to registry key `kling-v3-standard`.
|
|
925
|
+
// Veo: veo-3.1-{fast|standard}[-4k]-{N}s[-audio]
|
|
926
|
+
// Seedance: seedance-2-{fast|std}[-priority]-{N}s. Economy (PiAPI) and
|
|
927
|
+
// Priority (fal.ai) are DIFFERENT registry keys at DIFFERENT prices —
|
|
928
|
+
// priority is ~2.2x economy. The cost key MUST encode the tier or the
|
|
929
|
+
// pre-flight quote understates a priority gen (the desktop charges the
|
|
930
|
+
// priority key regardless of what we quoted).
|
|
931
|
+
const KLING_TIER_MAP = {
|
|
932
|
+
'kling-v3.0-std': 'kling-v3-standard',
|
|
933
|
+
'kling-v3.0-pro': 'kling-v3-pro',
|
|
934
|
+
'kling-v3.0-omni': 'kling-v3-omni',
|
|
935
|
+
};
|
|
936
|
+
function videoCostKey(input) {
|
|
937
|
+
if (input.model.startsWith('seedance')) {
|
|
938
|
+
// Mirrors seedanceCreditKey() in slate/src/shared/pricing.ts.
|
|
939
|
+
return input.seedanceSpeed === 'priority'
|
|
940
|
+
? `${input.model}-priority-${input.duration}s`
|
|
941
|
+
: `${input.model}-${input.duration}s`;
|
|
942
|
+
}
|
|
943
|
+
if (input.model.startsWith('veo')) {
|
|
944
|
+
const is4k = input.videoResolution === '4k';
|
|
945
|
+
const audio = input.sound !== false; // default audio on for Veo
|
|
946
|
+
const parts = [input.model];
|
|
947
|
+
if (is4k)
|
|
948
|
+
parts.push('4k');
|
|
949
|
+
parts.push(`${input.duration}s`);
|
|
950
|
+
if (audio)
|
|
951
|
+
parts.push('audio');
|
|
952
|
+
return parts.join('-');
|
|
953
|
+
}
|
|
954
|
+
if (input.model.startsWith('kling-v3.0')) {
|
|
955
|
+
const tier = KLING_TIER_MAP[input.model] ?? input.model;
|
|
956
|
+
return `${tier}-${input.duration}s`;
|
|
957
|
+
}
|
|
958
|
+
throw new Error(`Unknown video model: ${input.model}`);
|
|
959
|
+
}
|
|
960
|
+
// Maps a video model id to its bundled prompting skill (frontmatter `name:`),
|
|
961
|
+
// so guidance text points at a skill that actually exists. Deriving the name
|
|
962
|
+
// via model.split('-')[0] produced 'slates-prompting-kling' / '...-veo', which
|
|
963
|
+
// match no file — only seedance happened to line up.
|
|
964
|
+
function promptingSkillFor(model) {
|
|
965
|
+
if (model.startsWith('kling'))
|
|
966
|
+
return 'slates-prompting-kling-v3';
|
|
967
|
+
if (model.startsWith('veo'))
|
|
968
|
+
return 'slates-prompting-veo-3';
|
|
969
|
+
if (model.startsWith('seedance'))
|
|
970
|
+
return 'slates-prompting-seedance';
|
|
971
|
+
return 'slates-cost-discipline';
|
|
972
|
+
}
|
|
973
|
+
export const generateVideo = {
|
|
974
|
+
id: 'slates_generate_video',
|
|
975
|
+
description: 'Generate video via Slates credits. REQUIRED before calling: read the slates-cost-discipline skill plus the per-model prompting skill (slates-prompting-seedance / slates-prompting-kling-v3 / slates-prompting-veo-3) — video models prompt very differently. projectId is REQUIRED for UI integration (the user sees a progress card and the asset lands in the project — without it the call fails). aspectRatio + duration are required (server returns requires_clarification when missing). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. Veo locks to 16:9 and to 4/6/8s durations (4K only at 8s). Image-to-video via firstFrameAssetId. Frames-to-video via firstFrameAssetId + lastFrameAssetId (Veo / Seedance only). Ingredients via ingredientAssetIds (Kling Omni / Seedance). No skill files installed? Call slates_get_prompting_guide with the per-model guide (\'slates-prompting-veo-3\' / \'slates-prompting-kling-v3\' / \'slates-prompting-seedance\') and \'slates-cost-discipline\' before first use.',
|
|
976
|
+
input: z.object({
|
|
977
|
+
prompt: z.string().min(1).max(4000),
|
|
978
|
+
model: z.enum(VIDEO_MODELS).describe('Pick deliberately by capability AND cost. Kling V3.0 std = cheapest (no audio); pro = mid; omni = multi-char dialogue + audio. Veo 3.1 = top quality, locks 16:9, audio; fast vs standard. Seedance 2 = ByteDance, audio included, supports first+last frame; pick economy vs priority via seedanceSpeed. For exact per-call credit cost, call slates_estimate_generation_cost or slates_list_available_models — never quote prices from memory (they change).'),
|
|
979
|
+
projectId: z.string().uuid().optional().describe('Save into this Slates project. Strongly recommended — the desktop UI shows a progress card live and the asset appears when complete.'),
|
|
980
|
+
aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Veo locks to 16:9 — passing anything else will be ignored or fail. Kling/Seedance support all.'),
|
|
981
|
+
duration: z.number().int().min(4).max(15).optional().describe('Seconds. Kling: 5-15. Veo: 4, 6, or 8 only (4K only at 8s). Seedance: 4-15. Default 5 if omitted but always be explicit (cost scales linearly).'),
|
|
982
|
+
videoResolution: z.enum(['720p', '1080p', '4k']).optional().describe('Veo only. 720p / 1080p same price. 4k more expensive.'),
|
|
983
|
+
seedanceSpeed: z.enum(['economy', 'priority']).optional().describe('Seedance only. Economy via PiAPI (cheaper, slower). Priority via fal.ai (faster).'),
|
|
984
|
+
firstFrameAssetId: z.string().uuid().optional().describe('Asset id from the project — used as the starting frame for image-to-video. Must already exist in the project.'),
|
|
985
|
+
lastFrameAssetId: z.string().uuid().optional().describe('Asset id from the project — used as the ending frame. Veo and Seedance only. Pairs with firstFrameAssetId for guided transitions.'),
|
|
986
|
+
ingredientAssetIds: z.array(z.string().uuid()).max(9).optional().describe('Asset ids used as visual reference / ingredients for Kling Omni or Seedance. Up to 9 (Seedance) or 4 (Kling).'),
|
|
987
|
+
sound: z.boolean().optional().describe('Kling Omni / Veo / Seedance: enable audio generation. Default true.'),
|
|
988
|
+
audioLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('Kling Omni only — language for dialogue.'),
|
|
989
|
+
generateMusic: z.boolean().optional().describe('Kling Omni only — auto-generate background music.'),
|
|
990
|
+
negativePrompt: z.string().optional(),
|
|
991
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
992
|
+
confirm: z.boolean().optional().describe('Set true after explicit user OK to bypass the >$0.50 cost confirm gate (which fires for almost every video gen since they\'re expensive).'),
|
|
993
|
+
}),
|
|
994
|
+
async run(input, ctx) {
|
|
995
|
+
// projectId is required for video — without it there's no UI feedback,
|
|
996
|
+
// no asset to reference later, and a failed gen leaves the user with
|
|
997
|
+
// nothing. The MCP-only headless path that exists for image gen is
|
|
998
|
+
// not reasonable for video given the cost.
|
|
999
|
+
if (!input.projectId) {
|
|
1000
|
+
return ok({
|
|
1001
|
+
requires_clarification: true,
|
|
1002
|
+
missing: ['projectId'],
|
|
1003
|
+
message: 'projectId is required for video generation. Use slates_list_projects to find one or slates_create_project to make a new one. Video gens cost $0.50-$2.00+ per call — they need to land in a project so the user sees the progress card and the result.',
|
|
1004
|
+
});
|
|
1005
|
+
}
|
|
1006
|
+
if (!input.aspectRatio || !input.duration) {
|
|
1007
|
+
const missing = [];
|
|
1008
|
+
if (!input.aspectRatio)
|
|
1009
|
+
missing.push('aspectRatio');
|
|
1010
|
+
if (!input.duration)
|
|
1011
|
+
missing.push('duration');
|
|
1012
|
+
return ok({
|
|
1013
|
+
requires_clarification: true,
|
|
1014
|
+
missing,
|
|
1015
|
+
message: `Missing required field(s): ${missing.join(', ')}. ` +
|
|
1016
|
+
`Read the slates-cost-discipline + ${promptingSkillFor(input.model)} skills, ` +
|
|
1017
|
+
`or ask the user. Veo locks to 16:9. Kling/Seedance support 1:1 16:9 9:16 4:3 3:4 21:9. ` +
|
|
1018
|
+
`Duration: Kling 5-15s, Veo 4/6/8s (4K only at 8s), Seedance 4-15s. Cost scales linearly with duration.`,
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
// Veo exists only at discrete durations 4/6/8s, and 4K only at 8s.
|
|
1022
|
+
// Validate up front so the agent gets an actionable message instead of
|
|
1023
|
+
// a generic "Model variant not in registry" throw from the cost lookup.
|
|
1024
|
+
if (input.model.startsWith('veo')) {
|
|
1025
|
+
if (![4, 6, 8].includes(input.duration)) {
|
|
1026
|
+
return ok({
|
|
1027
|
+
requires_clarification: true,
|
|
1028
|
+
missing: ['duration'],
|
|
1029
|
+
message: `Veo 3.1 supports only 4s, 6s, or 8s (you passed ${input.duration}s). Pick one of those.`,
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
if (input.videoResolution === '4k' && input.duration !== 8) {
|
|
1033
|
+
return ok({
|
|
1034
|
+
requires_clarification: true,
|
|
1035
|
+
missing: ['duration'],
|
|
1036
|
+
message: `Veo 3.1 4K renders only at 8s (you passed ${input.duration}s at 4k). Use duration=8 for 4K, or drop to 720p/1080p for 4s/6s.`,
|
|
1037
|
+
});
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
const cloud = ctx.cloud();
|
|
1041
|
+
const registry = await cloud.get('/api/agent/models');
|
|
1042
|
+
const costKey = videoCostKey({
|
|
1043
|
+
model: input.model,
|
|
1044
|
+
duration: input.duration,
|
|
1045
|
+
videoResolution: input.videoResolution,
|
|
1046
|
+
sound: input.sound,
|
|
1047
|
+
seedanceSpeed: input.seedanceSpeed,
|
|
1048
|
+
});
|
|
1049
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
1050
|
+
if (!entry) {
|
|
1051
|
+
throw new Error(`Model variant not in registry: ${costKey}. ` +
|
|
1052
|
+
`Available video models: ${registry.models.filter((m) => m.model.startsWith('kling') || m.model.startsWith('veo') || m.model.startsWith('seedance')).map((m) => m.model).slice(0, 20).join(', ')}`);
|
|
1053
|
+
}
|
|
1054
|
+
const totalCents = entry.cost_cents;
|
|
1055
|
+
// Pre-flight confirm gate. Fires when:
|
|
1056
|
+
// (a) cost > $0.50 (the cost gate), OR
|
|
1057
|
+
// (b) any reference assets are involved (the look-first gate)
|
|
1058
|
+
// When references are present, the response inlines them as image
|
|
1059
|
+
// content blocks (and video keyframes for video refs) so the LLM
|
|
1060
|
+
// literally sees what it's about to use before committing spend.
|
|
1061
|
+
// The text steers the LLM to discuss the refs with the user by their
|
|
1062
|
+
// code+label so the conversation maps onto the gallery badges.
|
|
1063
|
+
const referenceRefs = [];
|
|
1064
|
+
if (input.firstFrameAssetId) {
|
|
1065
|
+
referenceRefs.push({ id: input.firstFrameAssetId, type: 'image', role: 'first frame' });
|
|
1066
|
+
}
|
|
1067
|
+
if (input.lastFrameAssetId) {
|
|
1068
|
+
referenceRefs.push({ id: input.lastFrameAssetId, type: 'image', role: 'last frame' });
|
|
1069
|
+
}
|
|
1070
|
+
for (const id of input.ingredientAssetIds ?? []) {
|
|
1071
|
+
referenceRefs.push({ id, type: 'image', role: 'ingredient' });
|
|
1072
|
+
}
|
|
1073
|
+
const hasReferences = referenceRefs.length > 0;
|
|
1074
|
+
if ((totalCents > 50 || hasReferences) && !input.confirm) {
|
|
1075
|
+
const previews = hasReferences ? await previewAssets(ctx, referenceRefs) : [];
|
|
1076
|
+
const refLines = previews.map((p) => ` - ${p.role}: ${p.ref}`).join('\n');
|
|
1077
|
+
const refSummary = hasReferences
|
|
1078
|
+
? `\n\nReferences attached above (in order — first frame, last frame, then ingredients):\n${refLines}\n\nReview them against your prompt. If the references suggest a different motion / framing / ` +
|
|
1079
|
+
`style than the current prompt captures, REVISE the prompt before confirming. When you talk to the user about this gen, refer to each reference by its code (e.g. "${previews[0]?.ref ?? 'IMG-A?'}") — they'll see the matching badge in the Slates gallery.`
|
|
1080
|
+
: '';
|
|
1081
|
+
const refImages = previews.flatMap((p) => p.images);
|
|
1082
|
+
return {
|
|
1083
|
+
text: `Pre-flight for ${input.duration}s ${input.model} (${costKey}): ` +
|
|
1084
|
+
`$${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
|
|
1085
|
+
refSummary +
|
|
1086
|
+
`\n\nWhen ready, re-call slates_generate_video with confirm=true and the (possibly revised) prompt.`,
|
|
1087
|
+
images: refImages,
|
|
1088
|
+
data: {
|
|
1089
|
+
requires_confirm: true,
|
|
1090
|
+
model: input.model,
|
|
1091
|
+
variant: costKey,
|
|
1092
|
+
estimated_cents: totalCents,
|
|
1093
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1094
|
+
references: previews.map((p) => ({
|
|
1095
|
+
role: p.role,
|
|
1096
|
+
ref: p.ref,
|
|
1097
|
+
asset_id: p.meta.asset_id,
|
|
1098
|
+
code: p.meta.code,
|
|
1099
|
+
label: p.meta.label,
|
|
1100
|
+
type: p.meta.type,
|
|
1101
|
+
frame_count: p.meta.frame_count,
|
|
1102
|
+
})),
|
|
1103
|
+
},
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
const desktop = ctx.desktop();
|
|
1107
|
+
if (input.background) {
|
|
1108
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1109
|
+
}
|
|
1110
|
+
const result = await desktop.post('/agent/generation/video', {
|
|
1111
|
+
projectId: input.projectId,
|
|
1112
|
+
model: input.model,
|
|
1113
|
+
prompt: input.prompt,
|
|
1114
|
+
aspectRatio: input.aspectRatio,
|
|
1115
|
+
duration: input.duration,
|
|
1116
|
+
videoResolution: input.videoResolution,
|
|
1117
|
+
seedanceSpeed: input.seedanceSpeed,
|
|
1118
|
+
firstFrameAssetId: input.firstFrameAssetId,
|
|
1119
|
+
lastFrameAssetId: input.lastFrameAssetId,
|
|
1120
|
+
ingredientAssetIds: input.ingredientAssetIds ?? [],
|
|
1121
|
+
sound: input.sound,
|
|
1122
|
+
audioLanguage: input.audioLanguage,
|
|
1123
|
+
generateMusic: input.generateMusic,
|
|
1124
|
+
negativePrompt: input.negativePrompt,
|
|
1125
|
+
background: input.background,
|
|
1126
|
+
});
|
|
1127
|
+
if (!result.success) {
|
|
1128
|
+
throw new Error(result.error ?? 'Generation failed');
|
|
1129
|
+
}
|
|
1130
|
+
if (result.background) {
|
|
1131
|
+
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1132
|
+
return backgroundSubmitted(`${input.duration}s ${input.model} video generation`, ids, {
|
|
1133
|
+
model: input.model,
|
|
1134
|
+
variant: costKey,
|
|
1135
|
+
projectId: input.projectId,
|
|
1136
|
+
cost_cents: totalCents,
|
|
1137
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
return {
|
|
1141
|
+
text: `Generated ${input.duration}s ${input.model} video into project ${input.projectId} ` +
|
|
1142
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
1143
|
+
`Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
|
|
1144
|
+
data: {
|
|
1145
|
+
model: input.model,
|
|
1146
|
+
variant: costKey,
|
|
1147
|
+
projectId: input.projectId,
|
|
1148
|
+
aspectRatio: input.aspectRatio,
|
|
1149
|
+
duration: input.duration,
|
|
1150
|
+
cost_cents: totalCents,
|
|
1151
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1152
|
+
asset: result.asset,
|
|
1153
|
+
generationId: result.generationId,
|
|
1154
|
+
},
|
|
1155
|
+
};
|
|
1156
|
+
},
|
|
1157
|
+
};
|
|
1158
|
+
// ── Generate lip-sync ───────────────────────────────────────────
|
|
1159
|
+
export const generateLipSync = {
|
|
1160
|
+
id: 'slates_generate_lip_sync',
|
|
1161
|
+
description: 'Lip-sync a still image (avatar) or a video clip to audio via Kling. REQUIRED before calling: read the slates-cost-discipline + slates-prompting-lip-sync skills. projectId is REQUIRED — the source asset must already exist in the project. Two flows: (1) sourceType=video → lip-syncs an existing talking-head clip to new audio (~$0.11 / 5s, no confirm gate); (2) sourceType=image → animates a still portrait into a talking avatar (avatar-standard ~$0.42 / 5s; avatar-pro ~$0.86 / 5s, hits the >$0.50 confirm gate). Audio comes from either TTS (pass ttsText) or an uploaded file (pass audioFilePath). Always 5 seconds — Kling lip-sync does not support other durations. No skill files installed? Call slates_get_prompting_guide with \'slates-prompting-lip-sync\' (and \'slates-cost-discipline\') before first use.',
|
|
1162
|
+
input: z.object({
|
|
1163
|
+
projectId: z.string().uuid().describe('Slates project the source asset lives in. The new lip-synced video lands here.'),
|
|
1164
|
+
sourceAssetId: z.string().uuid().describe('Asset id of the still image (avatar flow) or video clip (lip-sync flow). Must already exist in the project — use slates_upload_reference_image or slates_generate_image / slates_generate_video first if needed.'),
|
|
1165
|
+
sourceType: z.enum(['image', 'video']).describe('"image" = animate a still portrait (avatar). "video" = re-sync an existing talking-head clip. Determines pricing — be deliberate.'),
|
|
1166
|
+
audioMethod: z.enum(['tts', 'upload']).describe('"tts" = generate speech from ttsText. "upload" = use the file at audioFilePath (absolute path on the user\'s machine).'),
|
|
1167
|
+
ttsText: z.string().min(1).max(2000).optional().describe('Required when audioMethod=tts. The exact words the avatar/clip will speak.'),
|
|
1168
|
+
ttsVoice: z.string().optional().describe('Kling voice id (e.g. "oversea_male1"). See slates-prompting-lip-sync skill for the voice catalog.'),
|
|
1169
|
+
ttsLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('TTS language. Default EN.'),
|
|
1170
|
+
ttsSpeed: z.number().min(0.5).max(2).optional().describe('TTS speech rate. Default 1.0. Range 0.5-2.0.'),
|
|
1171
|
+
audioFilePath: z.string().optional().describe('Required when audioMethod=upload. Absolute path to the audio file on the user\'s machine (mp3, wav, m4a).'),
|
|
1172
|
+
avatarModel: z.enum(['avatar-standard', 'avatar-pro']).optional().describe('Image-source only. avatar-standard ($0.42/5s) for general use. avatar-pro ($0.86/5s) for sharper face fidelity. Ignored when sourceType=video.'),
|
|
1173
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1174
|
+
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate. Required for avatar-pro.'),
|
|
1175
|
+
}),
|
|
1176
|
+
async run(input, ctx) {
|
|
1177
|
+
if (input.audioMethod === 'tts' && !input.ttsText) {
|
|
1178
|
+
return ok({
|
|
1179
|
+
requires_clarification: true,
|
|
1180
|
+
missing: ['ttsText'],
|
|
1181
|
+
message: 'audioMethod=tts requires ttsText. Pass the exact words the avatar/clip will speak.',
|
|
1182
|
+
});
|
|
1183
|
+
}
|
|
1184
|
+
if (input.audioMethod === 'upload' && !input.audioFilePath) {
|
|
1185
|
+
return ok({
|
|
1186
|
+
requires_clarification: true,
|
|
1187
|
+
missing: ['audioFilePath'],
|
|
1188
|
+
message: 'audioMethod=upload requires audioFilePath. Pass an absolute path to the audio file on the user\'s machine.',
|
|
1189
|
+
});
|
|
1190
|
+
}
|
|
1191
|
+
let costKey;
|
|
1192
|
+
if (input.sourceType === 'video') {
|
|
1193
|
+
costKey = 'kling-lip-sync-video-5s';
|
|
1194
|
+
}
|
|
1195
|
+
else {
|
|
1196
|
+
costKey = input.avatarModel === 'avatar-pro'
|
|
1197
|
+
? 'kling-lip-sync-avatar-pro-5s'
|
|
1198
|
+
: 'kling-lip-sync-avatar-5s';
|
|
1199
|
+
}
|
|
1200
|
+
const cloud = ctx.cloud();
|
|
1201
|
+
const registry = await cloud.get('/api/agent/models');
|
|
1202
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
1203
|
+
if (!entry)
|
|
1204
|
+
throw new Error(`Model variant not in registry: ${costKey}`);
|
|
1205
|
+
const totalCents = entry.cost_cents;
|
|
1206
|
+
// Cost confirm gate. Lip-sync is mechanical — the model re-syncs the
|
|
1207
|
+
// user-chosen source to the user-chosen audio. The agent doesn't
|
|
1208
|
+
// write a prompt that depends on what the source looks like, so we
|
|
1209
|
+
// skip the inline preview and just announce the source code in text.
|
|
1210
|
+
if (totalCents > 50 && !input.confirm) {
|
|
1211
|
+
const sourceRef = await lookupAssetRef(ctx.desktop(), input.sourceAssetId);
|
|
1212
|
+
const audioPreview = input.audioMethod === 'tts'
|
|
1213
|
+
? `Audio: TTS — "${(input.ttsText ?? '').slice(0, 120)}"`
|
|
1214
|
+
: `Audio: ${input.audioFilePath}`;
|
|
1215
|
+
return ok({
|
|
1216
|
+
requires_confirm: true,
|
|
1217
|
+
variant: costKey,
|
|
1218
|
+
estimated_cents: totalCents,
|
|
1219
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1220
|
+
source_ref: sourceRef,
|
|
1221
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${costKey}. ` +
|
|
1222
|
+
`Source: ${sourceRef}. ${audioPreview}. ` +
|
|
1223
|
+
`Re-call with confirm=true after the user explicitly OKs the spend. ` +
|
|
1224
|
+
`When discussing with the user, refer to the source by its code (matches the gallery badge).`,
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
const desktop = ctx.desktop();
|
|
1228
|
+
if (input.background) {
|
|
1229
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1230
|
+
}
|
|
1231
|
+
const result = await desktop.post('/agent/generation/lip-sync', {
|
|
1232
|
+
projectId: input.projectId,
|
|
1233
|
+
sourceAssetId: input.sourceAssetId,
|
|
1234
|
+
audioMethod: input.audioMethod,
|
|
1235
|
+
ttsText: input.ttsText,
|
|
1236
|
+
ttsVoice: input.ttsVoice,
|
|
1237
|
+
ttsLanguage: input.ttsLanguage,
|
|
1238
|
+
ttsSpeed: input.ttsSpeed,
|
|
1239
|
+
audioFilePath: input.audioFilePath,
|
|
1240
|
+
avatarModel: input.avatarModel,
|
|
1241
|
+
estimatedCost: totalCents,
|
|
1242
|
+
background: input.background,
|
|
1243
|
+
});
|
|
1244
|
+
if (!result.success)
|
|
1245
|
+
throw new Error(result.error ?? 'Lip-sync generation failed');
|
|
1246
|
+
if (result.background) {
|
|
1247
|
+
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1248
|
+
return backgroundSubmitted(`5s lip-sync (${costKey})`, ids, {
|
|
1249
|
+
variant: costKey,
|
|
1250
|
+
projectId: input.projectId,
|
|
1251
|
+
sourceAssetId: input.sourceAssetId,
|
|
1252
|
+
cost_cents: totalCents,
|
|
1253
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1254
|
+
});
|
|
1255
|
+
}
|
|
1256
|
+
return {
|
|
1257
|
+
text: `Generated 5s lip-sync (${costKey}) into project ${input.projectId} ` +
|
|
1258
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
|
|
1259
|
+
(input.audioMethod === 'tts'
|
|
1260
|
+
? `Spoken: "${(input.ttsText ?? '').slice(0, 60)}${(input.ttsText ?? '').length > 60 ? '...' : ''}"`
|
|
1261
|
+
: `Audio: ${input.audioFilePath}`),
|
|
1262
|
+
data: {
|
|
1263
|
+
variant: costKey,
|
|
1264
|
+
projectId: input.projectId,
|
|
1265
|
+
sourceType: input.sourceType,
|
|
1266
|
+
sourceAssetId: input.sourceAssetId,
|
|
1267
|
+
cost_cents: totalCents,
|
|
1268
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1269
|
+
asset: result.asset,
|
|
1270
|
+
generationId: result.generationId,
|
|
1271
|
+
},
|
|
1272
|
+
};
|
|
1273
|
+
},
|
|
1274
|
+
};
|
|
1275
|
+
// ── Generate motion transfer ────────────────────────────────────
|
|
1276
|
+
export const generateMotionTransfer = {
|
|
1277
|
+
id: 'slates_generate_motion_transfer',
|
|
1278
|
+
description: 'Transfer the motion from a reference video onto a target image character via Kling Motion Control. REQUIRED before calling: read the slates-cost-discipline + slates-prompting-motion-transfer skills. projectId is REQUIRED — both source video and target image must already exist as assets in the project. Two tiers: kling-mc-std ($0.95 / 5s) and kling-mc-pro ($1.26 / 5s) — both hit the >$0.50 confirm gate. Always 5 seconds. No skill files installed? Call slates_get_prompting_guide with \'slates-prompting-motion-transfer\' (and \'slates-cost-discipline\') before first use.',
|
|
1279
|
+
input: z.object({
|
|
1280
|
+
projectId: z.string().uuid().describe('Slates project. Both source and target assets must live here.'),
|
|
1281
|
+
sourceVideoAssetId: z.string().uuid().describe('Asset id of the reference video — its motion will be retargeted onto the target image. Must already exist in the project.'),
|
|
1282
|
+
targetImageAssetId: z.string().uuid().describe('Asset id of the target image (the character that will perform the motion). Must already exist in the project.'),
|
|
1283
|
+
motionModel: z.enum(['kling-mc-std', 'kling-mc-pro']).optional().describe('std ($0.95) for general motion. pro ($1.26) for cleaner anatomy + identity preservation. Default pro — pick std deliberately for cost savings.'),
|
|
1284
|
+
characterOrientation: z.enum(['video', 'image']).optional().describe('"video" = use the source video\'s framing. "image" = use the target image\'s framing. Default video.'),
|
|
1285
|
+
prompt: z.string().optional().describe('Optional refinement prompt. Read slates-prompting-motion-transfer for guidance.'),
|
|
1286
|
+
background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
|
|
1287
|
+
confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 confirm gate. Required — both tiers exceed.'),
|
|
1288
|
+
}),
|
|
1289
|
+
async run(input, ctx) {
|
|
1290
|
+
const motionModel = input.motionModel ?? 'kling-mc-pro';
|
|
1291
|
+
const costKey = motionModel === 'kling-mc-std' ? 'kling-mc-std-5s' : 'kling-mc-pro-5s';
|
|
1292
|
+
const cloud = ctx.cloud();
|
|
1293
|
+
const registry = await cloud.get('/api/agent/models');
|
|
1294
|
+
const entry = registry.models.find((m) => m.model === costKey);
|
|
1295
|
+
if (!entry)
|
|
1296
|
+
throw new Error(`Model variant not in registry: ${costKey}`);
|
|
1297
|
+
const totalCents = entry.cost_cents;
|
|
1298
|
+
// Cost confirm gate. Motion transfer is mechanical — the model
|
|
1299
|
+
// applies source motion to target image deterministically. We don't
|
|
1300
|
+
// burn tokens previewing assets the user already chose; codes in the
|
|
1301
|
+
// text are enough to keep the chat unambiguous.
|
|
1302
|
+
if (totalCents > 50 && !input.confirm) {
|
|
1303
|
+
const desktop = ctx.desktop();
|
|
1304
|
+
const [source, target] = await Promise.all([
|
|
1305
|
+
lookupAssetRef(desktop, input.sourceVideoAssetId),
|
|
1306
|
+
lookupAssetRef(desktop, input.targetImageAssetId),
|
|
1307
|
+
]);
|
|
1308
|
+
return ok({
|
|
1309
|
+
requires_confirm: true,
|
|
1310
|
+
variant: costKey,
|
|
1311
|
+
estimated_cents: totalCents,
|
|
1312
|
+
estimated_dollars: (totalCents / 100).toFixed(2),
|
|
1313
|
+
source_ref: source,
|
|
1314
|
+
target_ref: target,
|
|
1315
|
+
message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${motionModel}. ` +
|
|
1316
|
+
`Transferring motion from ${source} onto ${target}. ` +
|
|
1317
|
+
`Re-call with confirm=true after the user explicitly OKs the spend, or pick kling-mc-std to save $0.31. ` +
|
|
1318
|
+
`When discussing with the user, refer to the assets by those codes — they'll match the gallery badges.`,
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
const desktop = ctx.desktop();
|
|
1322
|
+
if (input.background) {
|
|
1323
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1324
|
+
}
|
|
1325
|
+
const result = await desktop.post('/agent/generation/motion-transfer', {
|
|
1326
|
+
projectId: input.projectId,
|
|
1327
|
+
sourceVideoAssetId: input.sourceVideoAssetId,
|
|
1328
|
+
targetImageAssetId: input.targetImageAssetId,
|
|
1329
|
+
motionModel,
|
|
1330
|
+
characterOrientation: input.characterOrientation ?? 'video',
|
|
1331
|
+
prompt: input.prompt,
|
|
1332
|
+
estimatedCost: totalCents,
|
|
1333
|
+
background: input.background,
|
|
1334
|
+
});
|
|
1335
|
+
if (!result.success)
|
|
1336
|
+
throw new Error(result.error ?? 'Motion transfer generation failed');
|
|
1337
|
+
if (result.background) {
|
|
1338
|
+
const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
|
|
1339
|
+
return backgroundSubmitted(`5s motion transfer (${motionModel})`, ids, {
|
|
1340
|
+
variant: costKey,
|
|
1341
|
+
motionModel,
|
|
1342
|
+
projectId: input.projectId,
|
|
1343
|
+
sourceVideoAssetId: input.sourceVideoAssetId,
|
|
1344
|
+
targetImageAssetId: input.targetImageAssetId,
|
|
1345
|
+
cost_cents: totalCents,
|
|
1346
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1347
|
+
});
|
|
1348
|
+
}
|
|
1349
|
+
return {
|
|
1350
|
+
text: `Generated 5s motion transfer (${motionModel}) into project ${input.projectId} ` +
|
|
1351
|
+
`for $${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
|
|
1352
|
+
(input.prompt ? ` Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` : ''),
|
|
1353
|
+
data: {
|
|
1354
|
+
variant: costKey,
|
|
1355
|
+
motionModel,
|
|
1356
|
+
projectId: input.projectId,
|
|
1357
|
+
sourceVideoAssetId: input.sourceVideoAssetId,
|
|
1358
|
+
targetImageAssetId: input.targetImageAssetId,
|
|
1359
|
+
cost_cents: totalCents,
|
|
1360
|
+
cost_dollars: (totalCents / 100).toFixed(2),
|
|
1361
|
+
asset: result.asset,
|
|
1362
|
+
generationId: result.generationId,
|
|
1363
|
+
},
|
|
1364
|
+
};
|
|
1365
|
+
},
|
|
1366
|
+
};
|
|
1367
|
+
// ── Generation status (background mode) ─────────────────────────
|
|
1368
|
+
export const getGenerationStatus = {
|
|
1369
|
+
id: 'slates_get_generation_status',
|
|
1370
|
+
description: 'Poll one generation by id. Returns status (pending/processing/completed/failed/cancelled), the error message on failure, and the finished asset record (with id, code, filePath) on completion. Use after submitting any generate_* op with background=true; poll every 5-15s — video generations commonly take 1-5 minutes. Generations survive app restarts (the desktop resumes in-flight provider jobs on boot).',
|
|
1371
|
+
input: z.object({ generationId: z.string().uuid() }),
|
|
1372
|
+
async run(input, ctx) {
|
|
1373
|
+
const desktop = ctx.desktop();
|
|
1374
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1375
|
+
return ok(await desktop.get('/agent/generation/status', { id: input.generationId }));
|
|
1376
|
+
},
|
|
1377
|
+
};
|
|
1378
|
+
export const listGenerations = {
|
|
1379
|
+
id: 'slates_list_generations',
|
|
1380
|
+
description: "List recent and in-flight generations (newest first), optionally filtered by project and/or status. Use status='processing' to see everything still running, or no filter to review the recent history with costs and errors.",
|
|
1381
|
+
input: z.object({
|
|
1382
|
+
projectId: z.string().uuid().optional(),
|
|
1383
|
+
status: z.enum(['pending', 'processing', 'completed', 'failed', 'cancelled']).optional(),
|
|
1384
|
+
limit: z.number().int().min(1).max(100).optional(),
|
|
1385
|
+
}),
|
|
1386
|
+
async run(input, ctx) {
|
|
1387
|
+
const desktop = ctx.desktop();
|
|
1388
|
+
await desktop.requireCapability('background-generation', 'background generation');
|
|
1389
|
+
return ok(await desktop.get('/agent/generation/list', {
|
|
1390
|
+
projectId: input.projectId,
|
|
1391
|
+
status: input.status,
|
|
1392
|
+
limit: input.limit,
|
|
1393
|
+
}));
|
|
1394
|
+
},
|
|
1395
|
+
};
|
|
1396
|
+
export const getTimeline = {
|
|
1397
|
+
id: 'slates_get_timeline',
|
|
1398
|
+
description: 'Get (or lazily create) the single editing timeline for a Slates project, with all tracks, clips, markers, and a flat clipIndex mapping every clip back to its source asset (assetId + code + label). Frames are the unit of time; durationSec is provided. Call this before adding, reordering, or removing clips, and before exporting — it tells you the timeline id, frame rate, resolution, and current end frame.',
|
|
1399
|
+
input: z.object({ projectId: z.string().uuid() }),
|
|
1400
|
+
async run(input, ctx) {
|
|
1401
|
+
const desktop = ctx.desktop();
|
|
1402
|
+
await desktop.requireCapability('timeline', 'timeline editing');
|
|
1403
|
+
const r = await desktop.get('/agent/timeline', {
|
|
1404
|
+
projectId: input.projectId,
|
|
1405
|
+
});
|
|
1406
|
+
const t = r.timeline ?? {};
|
|
1407
|
+
const clipCount = r.clipIndex?.length ??
|
|
1408
|
+
(t.tracks ?? []).reduce((n, tr) => n + (tr.clips?.length ?? 0), 0);
|
|
1409
|
+
const durationSec = t.durationSec ?? r.durationSec;
|
|
1410
|
+
const dims = t.width && t.height ? `${t.width}x${t.height}` : '?';
|
|
1411
|
+
return ok(r, `Timeline for project ${input.projectId}: ${clipCount} clip(s) on ${t.tracks?.length ?? '?'} track(s), ` +
|
|
1412
|
+
`${durationSec ?? '?'}s at ${t.frameRate ?? '?'} fps, ${dims}.\n\n` +
|
|
1413
|
+
JSON.stringify(r, null, 2));
|
|
1414
|
+
},
|
|
1415
|
+
};
|
|
1416
|
+
export const addClipToTimeline = {
|
|
1417
|
+
id: 'slates_add_clip_to_timeline',
|
|
1418
|
+
description: "Append a video asset from the project to the project's timeline (or place it at an explicit startFrame). Defaults match the desktop UI: clip goes to the end of the first video track, full source duration, and an empty timeline auto-adopts the clip's resolution and frame rate. Optionally trim with sourceInFrame/sourceOutFrame (frames at the SOURCE fps). Use slates_get_timeline first to see current clips and pick positions. Only video assets are accepted.",
|
|
1419
|
+
input: z.object({
|
|
1420
|
+
projectId: z.string().uuid(),
|
|
1421
|
+
assetId: z.string().uuid().describe('Video asset already in the project.'),
|
|
1422
|
+
trackId: z.string().uuid().optional().describe('Target track. Default: the first video track.'),
|
|
1423
|
+
startFrame: z.number().int().min(0).optional().describe('Timeline frame to place the clip at. Default: append after the last clip.'),
|
|
1424
|
+
sourceInFrame: z.number().int().min(0).optional(),
|
|
1425
|
+
sourceOutFrame: z.number().int().min(1).optional(),
|
|
1426
|
+
}),
|
|
1427
|
+
async run(input, ctx) {
|
|
1428
|
+
const desktop = ctx.desktop();
|
|
1429
|
+
await desktop.requireCapability('timeline', 'timeline editing');
|
|
1430
|
+
const r = await desktop.post('/agent/timeline/add-clip', {
|
|
1431
|
+
projectId: input.projectId,
|
|
1432
|
+
assetId: input.assetId,
|
|
1433
|
+
trackId: input.trackId,
|
|
1434
|
+
startFrame: input.startFrame,
|
|
1435
|
+
sourceInFrame: input.sourceInFrame,
|
|
1436
|
+
sourceOutFrame: input.sourceOutFrame,
|
|
1437
|
+
});
|
|
1438
|
+
const clip = r.clip ?? {};
|
|
1439
|
+
// Code + label for the chat reference: prefer fields on the clip, fall
|
|
1440
|
+
// back to the clipIndex entry for this clip / its source asset.
|
|
1441
|
+
const idx = (r.clipIndex ?? []).find((c) => (clip.id && c.clipId === clip.id) || (!clip.id && clip.assetId && c.assetId === clip.assetId));
|
|
1442
|
+
const code = clip.code ?? idx?.code ?? null;
|
|
1443
|
+
const label = clip.label ?? idx?.label ?? null;
|
|
1444
|
+
const ref = code ? (label ? `${code} — ${label}` : code) : input.assetId;
|
|
1445
|
+
const frames = clip.durationFrames ??
|
|
1446
|
+
(clip.endFrame != null && clip.startFrame != null ? clip.endFrame - clip.startFrame : '?');
|
|
1447
|
+
return ok(r, `Added ${ref} to timeline at frame ${clip.startFrame ?? '?'} (${frames} frames long).`);
|
|
1448
|
+
},
|
|
1449
|
+
};
|
|
1450
|
+
export const reorderClips = {
|
|
1451
|
+
id: 'slates_reorder_clips',
|
|
1452
|
+
description: "Reorder the clips on one timeline track. Pass the COMPLETE list of the track's clip ids in the desired playback order; clips are repacked back-to-back from frame 0 (existing gaps are removed). Get current clip ids from slates_get_timeline.",
|
|
1453
|
+
input: z.object({
|
|
1454
|
+
trackId: z.string().uuid(),
|
|
1455
|
+
clipIds: z.array(z.string().uuid()).min(1),
|
|
1456
|
+
}),
|
|
1457
|
+
async run(input, ctx) {
|
|
1458
|
+
const desktop = ctx.desktop();
|
|
1459
|
+
await desktop.requireCapability('timeline', 'timeline editing');
|
|
1460
|
+
return ok(await desktop.post('/agent/timeline/reorder-clips', input));
|
|
1461
|
+
},
|
|
1462
|
+
};
|
|
1463
|
+
export const removeClip = {
|
|
1464
|
+
id: 'slates_remove_clip',
|
|
1465
|
+
description: "Remove a clip from the timeline (the source asset is untouched). Leaves a gap at the clip's old position — MP4 export renders gaps as black; call slates_reorder_clips afterwards to close gaps.",
|
|
1466
|
+
input: z.object({ clipId: z.string().uuid() }),
|
|
1467
|
+
async run(input, ctx) {
|
|
1468
|
+
const desktop = ctx.desktop();
|
|
1469
|
+
await desktop.requireCapability('timeline', 'timeline editing');
|
|
1470
|
+
return ok(await desktop.post('/agent/timeline/remove-clip', input));
|
|
1471
|
+
},
|
|
1472
|
+
};
|
|
1473
|
+
// ── Export ──────────────────────────────────────────────────────
|
|
1474
|
+
const ABSOLUTE_PATH_RE = /^([A-Za-z]:[\\/]|\/)/;
|
|
1475
|
+
export const exportVideo = {
|
|
1476
|
+
id: 'slates_export_video',
|
|
1477
|
+
description: "Render the project's timeline to an MP4 file on disk via the desktop app's ffmpeg pipeline (H.264 + AAC, gaps rendered as black). No dialogs — pass an absolute outputPath ending in .mp4. Fails if the file exists unless overwrite=true. Blocks until encoding finishes (can take minutes for long timelines) and returns the real file size and probed duration. After success, consider slates_reveal_file to show the file to the user. Requires at least one video clip on the timeline.",
|
|
1478
|
+
input: z
|
|
1479
|
+
.object({
|
|
1480
|
+
projectId: z.string().uuid().optional(),
|
|
1481
|
+
timelineId: z.string().uuid().optional(),
|
|
1482
|
+
outputPath: z
|
|
1483
|
+
.string()
|
|
1484
|
+
.min(5)
|
|
1485
|
+
.refine((p) => ABSOLUTE_PATH_RE.test(p), { message: 'outputPath must be absolute' })
|
|
1486
|
+
.describe('Absolute path for the rendered file, ending in .mp4 (e.g. C:\\Users\\you\\Videos\\ad.mp4 or /Users/you/ad.mp4). slates_get_project_directory gives a sensible default folder.'),
|
|
1487
|
+
overwrite: z.boolean().optional(),
|
|
1488
|
+
})
|
|
1489
|
+
.refine((d) => !!d.projectId !== !!d.timelineId, {
|
|
1490
|
+
message: 'Pass exactly one of projectId or timelineId',
|
|
1491
|
+
}),
|
|
1492
|
+
async run(input, ctx) {
|
|
1493
|
+
const desktop = ctx.desktop();
|
|
1494
|
+
await desktop.requireCapability('export', 'video export');
|
|
1495
|
+
const r = await desktop.post('/agent/timeline/export-video', {
|
|
1496
|
+
projectId: input.projectId,
|
|
1497
|
+
timelineId: input.timelineId,
|
|
1498
|
+
outputPath: input.outputPath,
|
|
1499
|
+
overwrite: input.overwrite,
|
|
1500
|
+
});
|
|
1501
|
+
const mb = r.fileSizeBytes != null ? (r.fileSizeBytes / (1024 * 1024)).toFixed(1) : '?';
|
|
1502
|
+
return ok(r, `Exported ${r.durationSec ?? '?'}s MP4 (${mb} MB) to ${r.outputPath ?? input.outputPath}.`);
|
|
1503
|
+
},
|
|
1504
|
+
};
|
|
1505
|
+
export const exportTimelineXml = {
|
|
1506
|
+
id: 'slates_export_timeline_xml',
|
|
1507
|
+
description: "Export the project's timeline as FCP7/XMEML XML — the file DaVinci Resolve imports directly (File → Import → Timeline) to recreate the edit with references to the original clip media on disk. This is the 'open in DaVinci' handoff path. Pass an absolute outputPath ending in .xml; fails if the file exists unless overwrite=true.",
|
|
1508
|
+
input: z
|
|
1509
|
+
.object({
|
|
1510
|
+
projectId: z.string().uuid().optional(),
|
|
1511
|
+
timelineId: z.string().uuid().optional(),
|
|
1512
|
+
outputPath: z
|
|
1513
|
+
.string()
|
|
1514
|
+
.min(5)
|
|
1515
|
+
.refine((p) => ABSOLUTE_PATH_RE.test(p), { message: 'outputPath must be absolute' })
|
|
1516
|
+
.describe('Absolute path for the XML file, ending in .xml (e.g. C:\\Users\\you\\Videos\\ad.xml or /Users/you/ad.xml).'),
|
|
1517
|
+
overwrite: z.boolean().optional(),
|
|
1518
|
+
})
|
|
1519
|
+
.refine((d) => !!d.projectId !== !!d.timelineId, {
|
|
1520
|
+
message: 'Pass exactly one of projectId or timelineId',
|
|
1521
|
+
}),
|
|
1522
|
+
async run(input, ctx) {
|
|
1523
|
+
const desktop = ctx.desktop();
|
|
1524
|
+
await desktop.requireCapability('export', 'timeline XML export');
|
|
1525
|
+
const r = await desktop.post('/agent/timeline/export-xml', {
|
|
1526
|
+
projectId: input.projectId,
|
|
1527
|
+
timelineId: input.timelineId,
|
|
1528
|
+
outputPath: input.outputPath,
|
|
1529
|
+
overwrite: input.overwrite,
|
|
1530
|
+
});
|
|
1531
|
+
return ok(r, `Exported timeline XML to ${r.outputPath ?? input.outputPath}. Import in DaVinci Resolve via File → Import → Timeline.`);
|
|
1532
|
+
},
|
|
1533
|
+
};
|
|
1534
|
+
export const revealFile = {
|
|
1535
|
+
id: 'slates_reveal_file',
|
|
1536
|
+
description: 'Open the OS file manager (Explorer/Finder) with the given file selected — use after slates_export_video / slates_export_timeline_xml so the user can see the file you wrote. Absolute path required; the file must exist.',
|
|
1537
|
+
input: z.object({ path: z.string().min(3) }),
|
|
1538
|
+
async run(input, ctx) {
|
|
1539
|
+
const desktop = ctx.desktop();
|
|
1540
|
+
await desktop.requireCapability('export', 'revealing files in the file manager');
|
|
1541
|
+
return ok(await desktop.post('/agent/reveal-file', input));
|
|
1542
|
+
},
|
|
1543
|
+
};
|
|
1544
|
+
// ── CRUD completion (agent API v1 routes — no capability check) ─
|
|
1545
|
+
export const updateProject = {
|
|
1546
|
+
id: 'slates_update_project',
|
|
1547
|
+
description: 'Update a Slates project\'s name and/or description.',
|
|
1548
|
+
input: z.object({
|
|
1549
|
+
id: z.string().uuid(),
|
|
1550
|
+
name: z.string().min(1).max(120).optional(),
|
|
1551
|
+
description: z.string().optional(),
|
|
1552
|
+
}),
|
|
1553
|
+
async run(input, ctx) {
|
|
1554
|
+
return ok(await ctx.desktop().post('/agent/projects/update', {
|
|
1555
|
+
id: input.id,
|
|
1556
|
+
data: { name: input.name, description: input.description },
|
|
1557
|
+
}));
|
|
1558
|
+
},
|
|
1559
|
+
};
|
|
1560
|
+
export const deleteProject = {
|
|
1561
|
+
id: 'slates_delete_project',
|
|
1562
|
+
description: 'Delete a Slates project. DESTRUCTIVE — permanently removes the project with all its assets, storyboards, and media files. Requires confirm=true after explicit user OK.',
|
|
1563
|
+
input: z.object({
|
|
1564
|
+
id: z.string().uuid(),
|
|
1565
|
+
confirm: z.boolean().optional().describe('Set true only after the user explicitly OKs the deletion.'),
|
|
1566
|
+
}),
|
|
1567
|
+
async run(input, ctx) {
|
|
1568
|
+
if (!input.confirm) {
|
|
1569
|
+
return ok({
|
|
1570
|
+
requires_confirm: true,
|
|
1571
|
+
message: 'Deleting a project permanently removes all its assets, storyboards, and media files. Re-call with confirm=true after explicit user OK.',
|
|
1572
|
+
});
|
|
1573
|
+
}
|
|
1574
|
+
return ok(await ctx.desktop().post('/agent/projects/delete', { id: input.id }));
|
|
1575
|
+
},
|
|
1576
|
+
};
|
|
1577
|
+
export const getProjectDirectory = {
|
|
1578
|
+
id: 'slates_get_project_directory',
|
|
1579
|
+
description: 'Get the absolute on-disk folder of a Slates project — useful for choosing an export outputPath default (e.g. <dir>/exports/final.mp4).',
|
|
1580
|
+
input: z.object({ id: z.string().uuid() }),
|
|
1581
|
+
async run(input, ctx) {
|
|
1582
|
+
return ok(await ctx.desktop().get('/agent/projects/location', { id: input.id }));
|
|
1583
|
+
},
|
|
1584
|
+
};
|
|
1585
|
+
export const deleteAsset = {
|
|
1586
|
+
id: 'slates_delete_asset',
|
|
1587
|
+
description: 'Delete an asset from its project. Permanent — also deletes the media file from disk.',
|
|
1588
|
+
input: z.object({ id: z.string().uuid() }),
|
|
1589
|
+
async run(input, ctx) {
|
|
1590
|
+
return ok(await ctx.desktop().post('/agent/assets/delete', { id: input.id }));
|
|
1591
|
+
},
|
|
1592
|
+
};
|
|
1593
|
+
export const renameFolder = {
|
|
1594
|
+
id: 'slates_rename_folder',
|
|
1595
|
+
description: 'Rename an asset folder.',
|
|
1596
|
+
input: z.object({
|
|
1597
|
+
folderId: z.string().uuid(),
|
|
1598
|
+
name: z.string().min(1).max(120),
|
|
1599
|
+
}),
|
|
1600
|
+
async run(input, ctx) {
|
|
1601
|
+
return ok(await ctx.desktop().post('/agent/folders/rename', { id: input.folderId, name: input.name }));
|
|
1602
|
+
},
|
|
1603
|
+
};
|
|
1604
|
+
export const deleteFolder = {
|
|
1605
|
+
id: 'slates_delete_folder',
|
|
1606
|
+
description: 'Delete an asset folder.',
|
|
1607
|
+
input: z.object({ folderId: z.string().uuid() }),
|
|
1608
|
+
async run(input, ctx) {
|
|
1609
|
+
return ok(await ctx.desktop().post('/agent/folders/delete', { id: input.folderId }));
|
|
1610
|
+
},
|
|
1611
|
+
};
|
|
1612
|
+
export const setFolderCover = {
|
|
1613
|
+
id: 'slates_set_folder_cover',
|
|
1614
|
+
description: 'Set the cover image of an asset folder (or clear it with assetId=null).',
|
|
1615
|
+
input: z.object({
|
|
1616
|
+
folderId: z.string().uuid(),
|
|
1617
|
+
assetId: z.string().uuid().nullable(),
|
|
1618
|
+
}),
|
|
1619
|
+
async run(input, ctx) {
|
|
1620
|
+
return ok(await ctx.desktop().post('/agent/folders/set-cover', input));
|
|
1621
|
+
},
|
|
1622
|
+
};
|
|
1623
|
+
export const updateCharacter = {
|
|
1624
|
+
id: 'slates_update_character',
|
|
1625
|
+
description: 'Update a character\'s name, description, or style. (Turnaround / expression image slots have their own dedicated set_character_* ops.)',
|
|
1626
|
+
input: z.object({
|
|
1627
|
+
characterId: z.string().uuid(),
|
|
1628
|
+
name: z.string().min(1).max(120).optional(),
|
|
1629
|
+
description: z.string().optional(),
|
|
1630
|
+
style: z.enum(['realistic', 'anime', 'pixar', 'comic-book']).optional(),
|
|
1631
|
+
}),
|
|
1632
|
+
async run(input, ctx) {
|
|
1633
|
+
return ok(await ctx.desktop().post('/agent/characters/update', {
|
|
1634
|
+
id: input.characterId,
|
|
1635
|
+
data: { name: input.name, description: input.description, style: input.style },
|
|
1636
|
+
}));
|
|
1637
|
+
},
|
|
1638
|
+
};
|
|
1639
|
+
export const deleteCharacter = {
|
|
1640
|
+
id: 'slates_delete_character',
|
|
1641
|
+
description: 'Delete a character from its project.',
|
|
1642
|
+
input: z.object({ characterId: z.string().uuid() }),
|
|
1643
|
+
async run(input, ctx) {
|
|
1644
|
+
return ok(await ctx.desktop().post('/agent/characters/delete', { id: input.characterId }));
|
|
1645
|
+
},
|
|
1646
|
+
};
|
|
1647
|
+
export const updateEnvironment = {
|
|
1648
|
+
id: 'slates_update_environment',
|
|
1649
|
+
description: 'Update an environment\'s name, description, style, or bound grid asset (gridAssetId=null clears it).',
|
|
1650
|
+
input: z.object({
|
|
1651
|
+
environmentId: z.string().uuid(),
|
|
1652
|
+
name: z.string().min(1).max(120).optional(),
|
|
1653
|
+
description: z.string().optional(),
|
|
1654
|
+
style: z.enum(['realistic', 'anime', 'pixar', 'comic-book']).optional(),
|
|
1655
|
+
gridAssetId: z.string().uuid().nullable().optional(),
|
|
1656
|
+
}),
|
|
1657
|
+
async run(input, ctx) {
|
|
1658
|
+
return ok(await ctx.desktop().post('/agent/environments/update', {
|
|
1659
|
+
id: input.environmentId,
|
|
1660
|
+
data: {
|
|
1661
|
+
name: input.name,
|
|
1662
|
+
description: input.description,
|
|
1663
|
+
style: input.style,
|
|
1664
|
+
gridAssetId: input.gridAssetId,
|
|
1665
|
+
},
|
|
1666
|
+
}));
|
|
1667
|
+
},
|
|
1668
|
+
};
|
|
1669
|
+
export const deleteEnvironment = {
|
|
1670
|
+
id: 'slates_delete_environment',
|
|
1671
|
+
description: 'Delete an environment from its project.',
|
|
1672
|
+
input: z.object({ environmentId: z.string().uuid() }),
|
|
1673
|
+
async run(input, ctx) {
|
|
1674
|
+
return ok(await ctx.desktop().post('/agent/environments/delete', { id: input.environmentId }));
|
|
1675
|
+
},
|
|
1676
|
+
};
|
|
1677
|
+
export const listStyles = {
|
|
1678
|
+
id: 'slates_list_styles',
|
|
1679
|
+
description: 'List visual styles in a Slates project.',
|
|
1680
|
+
input: z.object({ projectId: z.string().uuid() }),
|
|
1681
|
+
async run(input, ctx) {
|
|
1682
|
+
return ok(await ctx.desktop().get('/agent/styles', { projectId: input.projectId }));
|
|
1683
|
+
},
|
|
1684
|
+
};
|
|
1685
|
+
export const createStyle = {
|
|
1686
|
+
id: 'slates_create_style',
|
|
1687
|
+
description: 'Create a new visual style in a Slates project.',
|
|
1688
|
+
input: z.object({
|
|
1689
|
+
projectId: z.string().uuid(),
|
|
1690
|
+
name: z.string().min(1).max(120),
|
|
1691
|
+
description: z.string().optional(),
|
|
1692
|
+
}),
|
|
1693
|
+
async run(input, ctx) {
|
|
1694
|
+
return ok(await ctx.desktop().post('/agent/styles', input));
|
|
1695
|
+
},
|
|
1696
|
+
};
|
|
1697
|
+
export const updateStyle = {
|
|
1698
|
+
id: 'slates_update_style',
|
|
1699
|
+
description: 'Update a style\'s name, description, or bound reference image (imageAssetId=null clears it).',
|
|
1700
|
+
input: z.object({
|
|
1701
|
+
styleId: z.string().uuid(),
|
|
1702
|
+
name: z.string().min(1).max(120).optional(),
|
|
1703
|
+
description: z.string().optional(),
|
|
1704
|
+
imageAssetId: z.string().uuid().nullable().optional(),
|
|
1705
|
+
}),
|
|
1706
|
+
async run(input, ctx) {
|
|
1707
|
+
return ok(await ctx.desktop().post('/agent/styles/update', {
|
|
1708
|
+
id: input.styleId,
|
|
1709
|
+
data: { name: input.name, description: input.description, imageAssetId: input.imageAssetId },
|
|
1710
|
+
}));
|
|
1711
|
+
},
|
|
1712
|
+
};
|
|
1713
|
+
export const deleteStyle = {
|
|
1714
|
+
id: 'slates_delete_style',
|
|
1715
|
+
description: 'Delete a visual style from its project.',
|
|
1716
|
+
input: z.object({ styleId: z.string().uuid() }),
|
|
1717
|
+
async run(input, ctx) {
|
|
1718
|
+
return ok(await ctx.desktop().post('/agent/styles/delete', { id: input.styleId }));
|
|
1719
|
+
},
|
|
1720
|
+
};
|
|
1721
|
+
export const updateStoryboard = {
|
|
1722
|
+
id: 'slates_update_storyboard',
|
|
1723
|
+
description: 'Update a storyboard\'s name and/or description.',
|
|
1724
|
+
input: z.object({
|
|
1725
|
+
storyboardId: z.string().uuid(),
|
|
1726
|
+
name: z.string().min(1).max(120).optional(),
|
|
1727
|
+
description: z.string().optional(),
|
|
1728
|
+
}),
|
|
1729
|
+
async run(input, ctx) {
|
|
1730
|
+
return ok(await ctx.desktop().post('/agent/storyboards/update', {
|
|
1731
|
+
id: input.storyboardId,
|
|
1732
|
+
data: { name: input.name, description: input.description },
|
|
1733
|
+
}));
|
|
1734
|
+
},
|
|
1735
|
+
};
|
|
1736
|
+
export const deleteStoryboard = {
|
|
1737
|
+
id: 'slates_delete_storyboard',
|
|
1738
|
+
description: 'Delete a storyboard with all its scenes and frames (the referenced assets are untouched).',
|
|
1739
|
+
input: z.object({ storyboardId: z.string().uuid() }),
|
|
1740
|
+
async run(input, ctx) {
|
|
1741
|
+
return ok(await ctx.desktop().post('/agent/storyboards/delete', { id: input.storyboardId }));
|
|
1742
|
+
},
|
|
1743
|
+
};
|
|
1744
|
+
export const updateScene = {
|
|
1745
|
+
id: 'slates_update_scene',
|
|
1746
|
+
description: 'Update a scene\'s name and/or position within its storyboard.',
|
|
1747
|
+
input: z.object({
|
|
1748
|
+
sceneId: z.string().uuid(),
|
|
1749
|
+
name: z.string().min(1).max(120).optional(),
|
|
1750
|
+
position: z.number().int().min(0).optional(),
|
|
1751
|
+
}),
|
|
1752
|
+
async run(input, ctx) {
|
|
1753
|
+
return ok(await ctx.desktop().post('/agent/scenes/update', {
|
|
1754
|
+
id: input.sceneId,
|
|
1755
|
+
data: { name: input.name, position: input.position },
|
|
1756
|
+
}));
|
|
1757
|
+
},
|
|
1758
|
+
};
|
|
1759
|
+
export const deleteScene = {
|
|
1760
|
+
id: 'slates_delete_scene',
|
|
1761
|
+
description: 'Delete a scene (and its frames) from a storyboard.',
|
|
1762
|
+
input: z.object({ sceneId: z.string().uuid() }),
|
|
1763
|
+
async run(input, ctx) {
|
|
1764
|
+
return ok(await ctx.desktop().post('/agent/scenes/delete', { id: input.sceneId }));
|
|
1765
|
+
},
|
|
1766
|
+
};
|
|
1767
|
+
export const reorderScenes = {
|
|
1768
|
+
id: 'slates_reorder_scenes',
|
|
1769
|
+
description: 'Reorder the scenes of a storyboard. Pass the COMPLETE list of the storyboard\'s scene ids in the desired order.',
|
|
1770
|
+
input: z.object({
|
|
1771
|
+
storyboardId: z.string().uuid(),
|
|
1772
|
+
sceneIds: z.array(z.string().uuid()).min(1),
|
|
1773
|
+
}),
|
|
1774
|
+
async run(input, ctx) {
|
|
1775
|
+
return ok(await ctx.desktop().post('/agent/scenes/reorder', input));
|
|
1776
|
+
},
|
|
1777
|
+
};
|
|
1778
|
+
export const updateFrame = {
|
|
1779
|
+
id: 'slates_update_frame',
|
|
1780
|
+
description: 'Update a frame: shot label, notes, bound asset (assetId=null unbinds), scene, position, frameType (first/last/ingredient, null clears), or motion prompt (null clears).',
|
|
1781
|
+
input: z.object({
|
|
1782
|
+
frameId: z.string().uuid(),
|
|
1783
|
+
shotLabel: z.string().optional(),
|
|
1784
|
+
notes: z.string().optional(),
|
|
1785
|
+
assetId: z.string().uuid().nullable().optional(),
|
|
1786
|
+
sceneId: z.string().uuid().nullable().optional(),
|
|
1787
|
+
position: z.number().int().min(0).optional(),
|
|
1788
|
+
frameType: z.enum(['first', 'last', 'ingredient']).nullable().optional(),
|
|
1789
|
+
motionPrompt: z.string().nullable().optional(),
|
|
1790
|
+
}),
|
|
1791
|
+
async run(input, ctx) {
|
|
1792
|
+
return ok(await ctx.desktop().post('/agent/frames/update', {
|
|
1793
|
+
id: input.frameId,
|
|
1794
|
+
data: {
|
|
1795
|
+
shotLabel: input.shotLabel,
|
|
1796
|
+
notes: input.notes,
|
|
1797
|
+
assetId: input.assetId,
|
|
1798
|
+
sceneId: input.sceneId,
|
|
1799
|
+
position: input.position,
|
|
1800
|
+
frameType: input.frameType,
|
|
1801
|
+
motionPrompt: input.motionPrompt,
|
|
1802
|
+
},
|
|
1803
|
+
}));
|
|
1804
|
+
},
|
|
1805
|
+
};
|
|
1806
|
+
export const deleteFrame = {
|
|
1807
|
+
id: 'slates_delete_frame',
|
|
1808
|
+
description: 'Delete a frame from its scene (the referenced asset is untouched).',
|
|
1809
|
+
input: z.object({ frameId: z.string().uuid() }),
|
|
1810
|
+
async run(input, ctx) {
|
|
1811
|
+
return ok(await ctx.desktop().post('/agent/frames/delete', { id: input.frameId }));
|
|
1812
|
+
},
|
|
1813
|
+
};
|
|
1814
|
+
// ── Prompting guides (local lookup — no transport) ──────────────
|
|
1815
|
+
// Model-id → guide-name aliasing. Order matters: kling-mc-* (motion
|
|
1816
|
+
// transfer) must match before the generic kling-v3* check.
|
|
1817
|
+
function resolveGuideTopic(topic) {
|
|
1818
|
+
const t = topic.trim().toLowerCase();
|
|
1819
|
+
if (SKILLS[t])
|
|
1820
|
+
return t;
|
|
1821
|
+
if (t.startsWith('nano-banana'))
|
|
1822
|
+
return 'slates-prompting-nano-banana-2';
|
|
1823
|
+
if (t.startsWith('flux'))
|
|
1824
|
+
return 'slates-prompting-flux-2-max';
|
|
1825
|
+
if (t.startsWith('seedream'))
|
|
1826
|
+
return 'slates-prompting-seedream-5-lite';
|
|
1827
|
+
if (t.startsWith('veo'))
|
|
1828
|
+
return 'slates-prompting-veo-3';
|
|
1829
|
+
if (t.startsWith('kling-mc'))
|
|
1830
|
+
return 'slates-prompting-motion-transfer';
|
|
1831
|
+
if (t.startsWith('kling-v3'))
|
|
1832
|
+
return 'slates-prompting-kling-v3';
|
|
1833
|
+
if (t.startsWith('seedance'))
|
|
1834
|
+
return 'slates-prompting-seedance';
|
|
1835
|
+
if (t.startsWith('avatar-') || t.includes('lip-sync'))
|
|
1836
|
+
return 'slates-prompting-lip-sync';
|
|
1837
|
+
return null;
|
|
1838
|
+
}
|
|
1839
|
+
export const getPromptingGuide = {
|
|
1840
|
+
id: 'slates_get_prompting_guide',
|
|
1841
|
+
description: "Return the full markdown of a bundled Slates prompting/workflow guide. MCP-only clients (Claude Desktop, Smithery) don't get the CLI-installed skill files — call this instead. Accepts a guide name or a model id (e.g. 'veo-3.1-fast', 'kling-v3.0-pro', 'seedance-2-std', 'nano-banana-2') which maps to the right guide. ALWAYS read 'slates-cost-discipline' plus the relevant model guide before your first generation in a session.",
|
|
1842
|
+
input: z.object({
|
|
1843
|
+
topic: z
|
|
1844
|
+
.string()
|
|
1845
|
+
.min(1)
|
|
1846
|
+
.describe('Guide name or model id. Guides: slates-cost-discipline, slates-prompting-nano-banana-2, slates-prompting-veo-3, slates-prompting-kling-v3, slates-prompting-seedance, slates-prompting-lip-sync, slates-prompting-motion-transfer, slates-prompting-flux-2-max, slates-prompting-seedream-5-lite, slates-edit-and-iterate, slates-vision-feedback-loop, slates-character-turnaround, slates-storyboard-from-script, slates-direct-response-ad, slates-one-prompt-film'),
|
|
1847
|
+
}),
|
|
1848
|
+
async run(input) {
|
|
1849
|
+
const resolved = resolveGuideTopic(input.topic);
|
|
1850
|
+
const content = resolved ? SKILLS[resolved] : undefined;
|
|
1851
|
+
if (!resolved || content === undefined) {
|
|
1852
|
+
throw new Error(`Unknown guide topic: ${input.topic}. Valid topics: ${Object.keys(SKILLS).sort().join(', ')}`);
|
|
1853
|
+
}
|
|
1854
|
+
return {
|
|
1855
|
+
text: content,
|
|
1856
|
+
data: { topic: resolved, bytes: Buffer.byteLength(content, 'utf8') },
|
|
414
1857
|
};
|
|
415
1858
|
},
|
|
416
1859
|
};
|
|
@@ -426,6 +1869,8 @@ export const ALL_OPERATIONS = [
|
|
|
426
1869
|
getProject,
|
|
427
1870
|
listAssets,
|
|
428
1871
|
getAssetImage,
|
|
1872
|
+
getAssetsBatch,
|
|
1873
|
+
getAssetVideoFrames,
|
|
429
1874
|
uploadReferenceImage,
|
|
430
1875
|
listFolders,
|
|
431
1876
|
createFolder,
|
|
@@ -442,5 +1887,41 @@ export const ALL_OPERATIONS = [
|
|
|
442
1887
|
addScene,
|
|
443
1888
|
addFrame,
|
|
444
1889
|
generateImage,
|
|
1890
|
+
generateVideo,
|
|
1891
|
+
generateLipSync,
|
|
1892
|
+
generateMotionTransfer,
|
|
1893
|
+
editImage,
|
|
1894
|
+
getGenerationStatus,
|
|
1895
|
+
listGenerations,
|
|
1896
|
+
getTimeline,
|
|
1897
|
+
addClipToTimeline,
|
|
1898
|
+
reorderClips,
|
|
1899
|
+
removeClip,
|
|
1900
|
+
exportVideo,
|
|
1901
|
+
exportTimelineXml,
|
|
1902
|
+
revealFile,
|
|
1903
|
+
updateProject,
|
|
1904
|
+
deleteProject,
|
|
1905
|
+
getProjectDirectory,
|
|
1906
|
+
deleteAsset,
|
|
1907
|
+
renameFolder,
|
|
1908
|
+
deleteFolder,
|
|
1909
|
+
setFolderCover,
|
|
1910
|
+
updateCharacter,
|
|
1911
|
+
deleteCharacter,
|
|
1912
|
+
updateEnvironment,
|
|
1913
|
+
deleteEnvironment,
|
|
1914
|
+
listStyles,
|
|
1915
|
+
createStyle,
|
|
1916
|
+
updateStyle,
|
|
1917
|
+
deleteStyle,
|
|
1918
|
+
updateStoryboard,
|
|
1919
|
+
deleteStoryboard,
|
|
1920
|
+
updateScene,
|
|
1921
|
+
deleteScene,
|
|
1922
|
+
reorderScenes,
|
|
1923
|
+
updateFrame,
|
|
1924
|
+
deleteFrame,
|
|
1925
|
+
getPromptingGuide,
|
|
445
1926
|
];
|
|
446
1927
|
//# sourceMappingURL=index.js.map
|