@slatesvideo/shared 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +5 -0
  2. package/dist/clients/cloud.js +32 -1
  3. package/dist/clients/desktop.d.ts +10 -4
  4. package/dist/clients/desktop.js +95 -7
  5. package/dist/index.d.ts +2 -1
  6. package/dist/index.js +1 -0
  7. package/dist/operations/index.d.ts +207 -1
  8. package/dist/operations/index.js +1513 -32
  9. package/dist/prompts/character-sheet.d.ts +24 -0
  10. package/dist/prompts/character-sheet.js +53 -0
  11. package/dist/prompts/environment-sheet.d.ts +9 -0
  12. package/dist/prompts/environment-sheet.js +27 -0
  13. package/dist/prompts/index.d.ts +6 -0
  14. package/dist/prompts/index.js +17 -0
  15. package/dist/prompts/model-facts.d.ts +17 -0
  16. package/dist/prompts/model-facts.js +64 -0
  17. package/dist/prompts/reference-rules.d.ts +34 -0
  18. package/dist/prompts/reference-rules.js +120 -0
  19. package/dist/prompts/style-library.d.ts +31 -0
  20. package/dist/prompts/style-library.js +84 -0
  21. package/dist/skills/content.d.ts +2 -0
  22. package/dist/skills/content.js +20 -0
  23. package/package.json +36 -5
  24. package/skills/slates-character-turnaround.md +55 -0
  25. package/skills/slates-cost-discipline.md +96 -0
  26. package/skills/slates-direct-response-ad.md +66 -0
  27. package/skills/slates-edit-and-iterate.md +48 -0
  28. package/skills/slates-one-prompt-film.md +67 -0
  29. package/skills/slates-prompting-flux-2-max.md +112 -0
  30. package/skills/slates-prompting-kling-v3.md +167 -0
  31. package/skills/slates-prompting-lip-sync.md +169 -0
  32. package/skills/slates-prompting-motion-transfer.md +148 -0
  33. package/skills/slates-prompting-nano-banana-2.md +154 -0
  34. package/skills/slates-prompting-seedance.md +155 -0
  35. package/skills/slates-prompting-seedream-5-lite.md +74 -0
  36. package/skills/slates-prompting-veo-3.md +156 -0
  37. package/skills/slates-storyboard-from-script.md +49 -0
  38. package/skills/slates-vision-feedback-loop.md +62 -0
  39. package/dist/auth.d.ts.map +0 -1
  40. package/dist/auth.js.map +0 -1
  41. package/dist/clients/cloud.d.ts.map +0 -1
  42. package/dist/clients/cloud.js.map +0 -1
  43. package/dist/clients/desktop.d.ts.map +0 -1
  44. package/dist/clients/desktop.js.map +0 -1
  45. package/dist/index.d.ts.map +0 -1
  46. package/dist/index.js.map +0 -1
  47. package/dist/operations/index.d.ts.map +0 -1
  48. package/dist/operations/index.js.map +0 -1
@@ -12,6 +12,7 @@
12
12
  import { z } from 'zod';
13
13
  import { SlatesCloudClient } from '../clients/cloud.js';
14
14
  import { SlatesDesktopClient } from '../clients/desktop.js';
15
+ import { SKILLS } from '../skills/content.js';
15
16
  export function defaultContext() {
16
17
  return {
17
18
  cloud: () => new SlatesCloudClient(),
@@ -25,6 +26,21 @@ function ok(data, text) {
25
26
  data,
26
27
  };
27
28
  }
29
+ // Shared describe-text for the background flag on every generate_* op.
30
+ const BACKGROUND_DESCRIBE = 'Submit and return immediately with generationId(s) instead of blocking until the file is saved. ' +
31
+ 'Poll with slates_get_generation_status. Recommended for video (1-5 min renders).';
32
+ // Early-return shape when a generation route accepted the job in background
33
+ // mode ({ background: true } in the response). No inline-image fetch — the
34
+ // asset doesn't exist yet; the poller delivers it on completion.
35
+ function backgroundSubmitted(kind, ids, extra) {
36
+ const idText = ids.length > 0 ? ids.join(', ') : '(no id returned)';
37
+ return {
38
+ text: `Submitted ${kind} in the background — generationId(s): ${idText}. ` +
39
+ `Poll slates_get_generation_status with each id every 5-15s until status is 'completed' ` +
40
+ `(video generations commonly take 1-5 minutes). Generations survive app restarts.`,
41
+ data: { generationIds: ids, status: 'processing', ...extra },
42
+ };
43
+ }
28
44
  // ── Workspace + identity ────────────────────────────────────────
29
45
  export const getWorkspaceState = {
30
46
  id: 'slates_get_workspace_state',
@@ -135,17 +151,90 @@ export const listAssets = {
135
151
  };
136
152
  export const getAssetImage = {
137
153
  id: 'slates_get_asset_image',
138
- description: 'Read an asset image off disk and return it inline as base64. Used for vision: the calling LLM sees the actual pixels. Default JPEG 85% ≤1024px long-edge for token economy. Pass fullRes=true for the original PNG.',
154
+ description: 'Read an asset image off disk and return it inline as base64 so the calling LLM sees the actual pixels. The response also carries the asset\'s short code (e.g. IMG-A12) and human label — USE these when referring to the asset in chat with the user (e.g. "I\'m using IMG-A12 — Beach Sunset as the first frame") so they can match it to the badged thumbnail in the Slates gallery. Default JPEG 85% ≤1024px long-edge for token economy. Pass fullRes=true for the original PNG.',
139
155
  input: z.object({
140
156
  id: z.string().uuid(),
141
157
  fullRes: z.boolean().optional(),
142
158
  }),
143
159
  async run(input, ctx) {
144
160
  const r = await ctx.desktop().get('/agent/assets/image', { id: input.id, fullRes: input.fullRes ?? false });
161
+ const refLabel = r.code ? (r.label ? `${r.code} — ${r.label}` : r.code) : r.asset_id;
145
162
  return {
146
- text: `Loaded asset ${r.asset_id} (${r.bytes} bytes, ${r.mimeType}) from ${r.file_path}`,
163
+ text: `Loaded ${refLabel} (${r.bytes} bytes, ${r.mimeType}). When referring to this asset to the user, use "${refLabel}" — they can find the matching badge in the Slates gallery.`,
147
164
  images: [{ data: r.data, mimeType: r.mimeType }],
148
- data: { asset_id: r.asset_id, file_path: r.file_path, bytes: r.bytes, mimeType: r.mimeType },
165
+ data: {
166
+ asset_id: r.asset_id,
167
+ code: r.code,
168
+ label: r.label,
169
+ file_path: r.file_path,
170
+ bytes: r.bytes,
171
+ mimeType: r.mimeType,
172
+ },
173
+ };
174
+ },
175
+ };
176
+ export const getAssetsBatch = {
177
+ id: 'slates_get_assets_batch',
178
+ description: 'Fetch up to 8 image-asset thumbnails inline in a single call. Use this when picking the right reference from a project gallery — one round trip beats N. Each returned image carries its short code (IMG-A12) and label so you can speak about candidates in the user\'s shared vocabulary ("between IMG-A12 and IMG-A14, the second has the right composition"). Video assets are not supported here — call slates_get_asset_video_frames for those.',
179
+ input: z.object({
180
+ ids: z.array(z.string().uuid()).min(1).max(8).describe('1-8 image-asset ids. Order is preserved in the response.'),
181
+ }),
182
+ async run(input, ctx) {
183
+ const r = await ctx.desktop().post('/agent/assets/batch-images', { ids: input.ids });
184
+ const refs = r.images.map((i) => i.code ? (i.label ? `${i.code} — ${i.label}` : i.code) : i.asset_id);
185
+ const errSummary = r.errors.length > 0
186
+ ? ` Skipped ${r.errors.length}: ${r.errors.map((e) => `${e.asset_id} (${e.error})`).join('; ')}.`
187
+ : '';
188
+ return {
189
+ text: `Loaded ${r.images.length} image(s): ${refs.join(', ')}.` +
190
+ errSummary +
191
+ ` When discussing these with the user, reference them by their codes so they can match each one to a gallery badge.`,
192
+ images: r.images.map((i) => ({ data: i.data, mimeType: i.mimeType })),
193
+ data: {
194
+ images: r.images.map((i) => ({
195
+ asset_id: i.asset_id,
196
+ code: i.code,
197
+ label: i.label,
198
+ file_path: i.file_path,
199
+ bytes: i.bytes,
200
+ mimeType: i.mimeType,
201
+ })),
202
+ errors: r.errors,
203
+ },
204
+ };
205
+ },
206
+ };
207
+ export const getAssetVideoFrames = {
208
+ id: 'slates_get_asset_video_frames',
209
+ description: 'Extract N evenly-spaced keyframes from a video asset and return them inline as base64 JPEGs. This is the "see the video" path — LLMs can\'t consume video natively, so frames are the next best thing. Default 3 frames (start / middle / end). Bump to 5-8 for longer clips or when motion is the whole story. Use this before writing a motion-transfer prompt, a lip-sync refinement, or any iteration on a video clip. Response carries the asset\'s code (e.g. VID-V3) + label — name them when discussing the clip with the user.',
210
+ input: z.object({
211
+ id: z.string().uuid(),
212
+ count: z.number().int().min(1).max(8).optional().describe('Number of frames to extract. Default 3.'),
213
+ }),
214
+ async run(input, ctx) {
215
+ const params = { id: input.id };
216
+ if (input.count != null)
217
+ params.count = input.count;
218
+ const r = await ctx.desktop().get('/agent/assets/video-frames', params);
219
+ const refLabel = r.code ? (r.label ? `${r.code} — ${r.label}` : r.code) : r.asset_id;
220
+ const timestamps = r.frames.map((f) => `${f.timestamp_seconds}s`).join(', ');
221
+ return {
222
+ text: `Extracted ${r.frames.length} frame(s) from ${refLabel} ` +
223
+ `(${r.duration_seconds}s clip) at: ${timestamps}. ` +
224
+ `When referring to this video with the user, call it "${refLabel}".`,
225
+ images: r.frames.map((f) => ({ data: f.data, mimeType: f.mimeType })),
226
+ data: {
227
+ asset_id: r.asset_id,
228
+ code: r.code,
229
+ label: r.label,
230
+ file_path: r.file_path,
231
+ duration_seconds: r.duration_seconds,
232
+ frames: r.frames.map((f) => ({
233
+ timestamp_seconds: f.timestamp_seconds,
234
+ bytes: f.bytes,
235
+ mimeType: f.mimeType,
236
+ })),
237
+ },
149
238
  };
150
239
  },
151
240
  };
@@ -345,61 +434,334 @@ export const addFrame = {
345
434
  return ok(await ctx.desktop().post('/agent/frames', input));
346
435
  },
347
436
  };
437
+ function formatRef(asset) {
438
+ if (!asset.code)
439
+ return asset.asset_id;
440
+ return asset.label ? `${asset.code} — ${asset.label}` : asset.code;
441
+ }
442
+ // Cheap metadata-only lookup used by the mechanical-op confirm gates
443
+ // (motion_transfer, lip_sync). Returns the user-facing reference string
444
+ // without pulling any pixels — purely so the agent can announce the
445
+ // chosen assets to the user in shared vocabulary.
446
+ async function lookupAssetRef(desktop, id) {
447
+ try {
448
+ const r = await desktop.get('/agent/assets/path', { id });
449
+ return formatRef({ asset_id: id, code: r.code ?? null, label: r.label ?? null });
450
+ }
451
+ catch {
452
+ return id;
453
+ }
454
+ }
455
+ async function previewAsset(ctx, id, type, role, videoFrameCount = 3) {
456
+ const desktop = ctx.desktop();
457
+ if (type === 'image') {
458
+ const r = await desktop.get('/agent/assets/image', { id });
459
+ return {
460
+ ref: formatRef(r),
461
+ role,
462
+ images: [{ data: r.data, mimeType: r.mimeType }],
463
+ meta: {
464
+ asset_id: r.asset_id,
465
+ code: r.code,
466
+ label: r.label,
467
+ type: 'image',
468
+ file_path: r.file_path,
469
+ },
470
+ };
471
+ }
472
+ const r = await desktop.get('/agent/assets/video-frames', { id, count: videoFrameCount });
473
+ return {
474
+ ref: formatRef(r),
475
+ role,
476
+ images: r.frames.map((f) => ({ data: f.data, mimeType: f.mimeType })),
477
+ meta: {
478
+ asset_id: r.asset_id,
479
+ code: r.code,
480
+ label: r.label,
481
+ type: 'video',
482
+ file_path: r.file_path,
483
+ frame_count: r.frames.length,
484
+ },
485
+ };
486
+ }
487
+ async function previewAssets(ctx, refs) {
488
+ // Sequential — desktop is local, parallel buys nothing and the order
489
+ // matters for the confirm-gate text (first frame, last frame, ingredients...).
490
+ const out = [];
491
+ for (const r of refs) {
492
+ try {
493
+ out.push(await previewAsset(ctx, r.id, r.type, r.role));
494
+ }
495
+ catch (err) {
496
+ // Best-effort previews — a missing frame shouldn't block the gen,
497
+ // but flag it so the agent sees the gap.
498
+ out.push({
499
+ ref: `[unavailable] ${r.id}`,
500
+ role: r.role,
501
+ images: [],
502
+ meta: {
503
+ asset_id: r.id,
504
+ code: null,
505
+ label: null,
506
+ type: r.type,
507
+ file_path: '',
508
+ },
509
+ });
510
+ void err;
511
+ }
512
+ }
513
+ return out;
514
+ }
348
515
  // ── Generation (cloud-routed, credits-default) ──────────────────
516
+ // Registry cost-key for an image model+resolution. Mirrors imageCreditKey()
517
+ // in slate/src/shared/pricing.ts: NB2 prices per resolution, FLUX.2 Max prices
518
+ // per resolution (1k is the bare key), Seedream is flat (one key).
519
+ function imageCostKey(model, resolution) {
520
+ if (model === 'flux-2-max')
521
+ return resolution === '1k' ? 'flux-2-max' : `flux-2-max-${resolution}`;
522
+ if (model === 'seedream-5-lite')
523
+ return 'seedream-5-lite';
524
+ return `nano-banana-2-${resolution}`;
525
+ }
349
526
  export const generateImage = {
350
527
  id: 'slates_generate_image',
351
- description: 'Generate an image via Slates credits using the default Nano Banana 2 model. Returns inline image(s) so the calling LLM can evaluate them. Estimate cost first with slates_estimate_generation_cost. NOTE: MCP/CLI generation always charges credits BYOK is desktop-UI only by design.',
528
+ description: 'Generate an image via Slates credits. Three models: nano-banana-2 (Google Gemini 3 Image — default, strongest general image model, well-censored), flux-2-max (FLUX.2 Max photoreal, less censored, up to 4MP), seedream-5-lite (cheapest at ~$0.05 flat, less censored). Pass projectId to save into a Slates project (recommended — asset appears live in the desktop UI). FLUX.2 Max and Seedream 5 Lite REQUIRE projectId (no headless path). REQUIRED before calling: read the slates-cost-discipline skill (and slates-prompting-nano-banana-2 when using nano-banana-2). You MUST pass aspectRatio and resolution explicitly (the server returns requires_clarification when missing — defaults waste credits). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. MCP/CLI generation always charges credits. No skill files installed? Call slates_get_prompting_guide with the model\'s topic (and \'slates-cost-discipline\') before first use.',
352
529
  input: z.object({
353
530
  prompt: z.string().min(1).max(4000),
354
- resolution: z.enum(['1k', '2k', '4k']).optional(),
355
- aspectRatio: z.string().optional(),
531
+ model: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional().describe('Image model. Default nano-banana-2. Use flux-2-max for photoreal / less-censored, seedream-5-lite for cheapest. flux-2-max & seedream-5-lite require projectId.'),
532
+ projectId: z.string().uuid().optional().describe('Save into this Slates project. Renderer refreshes live. Required for flux-2-max / seedream-5-lite.'),
533
+ resolution: z.enum(['1k', '2k', '4k']).optional().describe('Pick deliberately: 1k drafts, 2k hero shots, 4k print/final. (Seedream is flat-priced regardless; FLUX & NB2 price by resolution.) Never default this.'),
534
+ aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Pick deliberately from the use case. Cinematic → 16:9. TikTok/Reels/Story → 9:16. IG square → 1:1. Ultra-wide → 21:9. Ask the user when ambiguous.'),
356
535
  count: z.number().int().min(1).max(4).optional(),
357
- referenceImageUrls: z.array(z.string().url()).max(14).optional(),
536
+ referenceImageUrls: z.array(z.string().url()).max(14).optional().describe('Headless (no-projectId) nano-banana-2 only: up to 14 ref URLs. For projectId runs, upload refs with slates_upload_reference_image first. Always label each image\'s role in the prompt text.'),
537
+ referenceAssetIds: z.array(z.string().uuid()).max(14).optional().describe("Project asset ids to use as reference/ingredient images (resolved on the desktop). Requires projectId. For nano-banana-2 up to 14 refs; FLUX/Seedream route to their edit endpoints with lower per-model caps. Label each reference's role in the prompt text."),
538
+ background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
358
539
  confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
359
540
  }),
360
541
  async run(input, ctx) {
361
- const resolution = input.resolution ?? '2k';
362
- const model = `nano-banana-2-${resolution}`;
542
+ // Clarification gate: aspectRatio + resolution must be deliberate.
543
+ // Mirrors the cost confirm gate — defaults silently wasted credits
544
+ // (1:1 when user wanted 16:9, 4k when 1k would have done). The LLM
545
+ // is forced to ask the user or read the skill instead of guessing.
546
+ if (!input.aspectRatio || !input.resolution) {
547
+ const missing = [];
548
+ if (!input.aspectRatio)
549
+ missing.push('aspectRatio');
550
+ if (!input.resolution)
551
+ missing.push('resolution');
552
+ return ok({
553
+ requires_clarification: true,
554
+ missing,
555
+ message: `Missing required field(s): ${missing.join(', ')}. ` +
556
+ `Read the slates-prompting-nano-banana-2 + slates-cost-discipline skills, ` +
557
+ `or ask the user. Aspect ratio options: 1:1 16:9 9:16 4:3 3:4 21:9 9:21 4:5 5:4 2:3 3:2. ` +
558
+ `Resolution options: 1k 2k 4k (same price band — pick by need, not cost).`,
559
+ });
560
+ }
561
+ const resolution = input.resolution;
562
+ const imageModel = input.model ?? 'nano-banana-2';
563
+ // FLUX.2 Max / Seedream 5 Lite have no headless path — they route through
564
+ // the desktop generation pipeline, which needs a project.
565
+ if (imageModel !== 'nano-banana-2' && !input.projectId) {
566
+ return ok({
567
+ requires_clarification: true,
568
+ missing: ['projectId'],
569
+ message: `${imageModel} requires projectId (no headless path). Use slates_list_projects / slates_create_project, then re-call with projectId.`,
570
+ });
571
+ }
572
+ // referenceAssetIds are PROJECT assets — the desktop resolves them off
573
+ // disk, so a headless run has nowhere to look them up. Same for
574
+ // background mode: the poller (slates_get_generation_status) reads the
575
+ // desktop's generation records.
576
+ const referenceAssetIds = input.referenceAssetIds ?? [];
577
+ if (referenceAssetIds.length > 0 && !input.projectId) {
578
+ return ok({
579
+ requires_clarification: true,
580
+ missing: ['projectId'],
581
+ message: 'referenceAssetIds are project assets resolved on the desktop — pass the projectId they live in (slates_list_projects to find it).',
582
+ });
583
+ }
584
+ if (input.background && !input.projectId) {
585
+ return ok({
586
+ requires_clarification: true,
587
+ missing: ['projectId'],
588
+ message: 'background=true routes through the desktop generation pipeline (so slates_get_generation_status can poll it) — pass a projectId, or drop background for a blocking headless run.',
589
+ });
590
+ }
591
+ if (referenceAssetIds.length > 0) {
592
+ await ctx.desktop().requireCapability('image-references', 'reference images on image generation');
593
+ }
594
+ if (input.background) {
595
+ await ctx.desktop().requireCapability('background-generation', 'background generation');
596
+ }
597
+ const costKey = imageCostKey(imageModel, resolution);
363
598
  const cloud = ctx.cloud();
364
599
  const registry = await cloud.get('/api/agent/models');
365
- const entry = registry.models.find((m) => m.model === model);
600
+ const entry = registry.models.find((m) => m.model === costKey);
366
601
  if (!entry)
367
- throw new Error(`Model not in registry: ${model}`);
602
+ throw new Error(`Model not in registry: ${costKey}`);
368
603
  const totalCents = entry.cost_cents * (input.count ?? 1);
369
- if (totalCents > 50 && !input.confirm) {
370
- return ok({
371
- requires_confirm: true,
372
- model,
373
- estimated_cents: totalCents,
374
- estimated_dollars: (totalCents / 100).toFixed(2),
375
- message: 'Cost exceeds $0.50. Re-call with confirm=true to proceed, or pick a smaller resolution / count.',
604
+ // Confirm gate. Fires on cost > $0.50, AND (look-first, mirroring
605
+ // slates_generate_video) whenever reference assets are involved
606
+ // regardless of cost — the LLM must see what it's referencing before
607
+ // committing spend.
608
+ if ((totalCents > 50 || referenceAssetIds.length > 0) && !input.confirm) {
609
+ if (referenceAssetIds.length === 0) {
610
+ return ok({
611
+ requires_confirm: true,
612
+ model: costKey,
613
+ estimated_cents: totalCents,
614
+ estimated_dollars: (totalCents / 100).toFixed(2),
615
+ message: 'Cost exceeds $0.50. Re-call with confirm=true to proceed, or pick a smaller resolution / count.',
616
+ });
617
+ }
618
+ const previews = await previewAssets(ctx, referenceAssetIds.map((id) => ({ id, type: 'image', role: 'reference' })));
619
+ const refLines = previews.map((p) => ` - ${p.role}: ${p.ref}`).join('\n');
620
+ return {
621
+ text: `Pre-flight for ${imageModel} (${costKey}): ` +
622
+ `$${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
623
+ `\n\nReference images attached above:\n${refLines}\n\n` +
624
+ `Review them against your prompt — every reference's role must be labeled in the prompt text. ` +
625
+ `If the references suggest a different composition / style than the current prompt captures, REVISE the prompt before confirming. ` +
626
+ `When you talk to the user about this gen, refer to each reference by its code (e.g. "${previews[0]?.ref ?? 'IMG-A?'}") — they'll see the matching badge in the Slates gallery.` +
627
+ `\n\nWhen ready, re-call slates_generate_image with confirm=true and the (possibly revised) prompt.`,
628
+ images: previews.flatMap((p) => p.images),
629
+ data: {
630
+ requires_confirm: true,
631
+ model: imageModel,
632
+ variant: costKey,
633
+ estimated_cents: totalCents,
634
+ estimated_dollars: (totalCents / 100).toFixed(2),
635
+ references: previews.map((p) => ({
636
+ role: p.role,
637
+ ref: p.ref,
638
+ asset_id: p.meta.asset_id,
639
+ code: p.meta.code,
640
+ label: p.meta.label,
641
+ type: p.meta.type,
642
+ })),
643
+ },
644
+ };
645
+ }
646
+ // Two paths:
647
+ // 1. projectId given → route through the desktop's normal generation
648
+ // pipeline. Eric sees the progress card light up, the asset save
649
+ // lands in the project, the gallery refreshes, and every
650
+ // renderer-side hook fires identically to a UI-triggered run.
651
+ // 2. no projectId → run headless via the cloud proxy and return
652
+ // bytes inline only (useful for headless agents that don't have
653
+ // a desktop attached).
654
+ if (input.projectId) {
655
+ const desktop = ctx.desktop();
656
+ const result = await desktop.post('/agent/generation/image', {
657
+ projectId: input.projectId,
658
+ prompt: input.prompt,
659
+ model: imageModel,
660
+ resolution,
661
+ aspectRatio: input.aspectRatio ?? '1:1',
662
+ count: input.count ?? 1,
663
+ ...(referenceAssetIds.length > 0 ? { referenceAssetIds } : {}),
664
+ background: input.background,
376
665
  });
666
+ // Partial multi-image failure: the desktop attaches an error message
667
+ // but result.assets still holds the images that DID save. Throwing
668
+ // here would discard real, paid-for assets and invite a full retry
669
+ // that double-spends — surface what landed instead.
670
+ const partialFailure = !result.success && Array.isArray(result.assets) && result.assets.length > 0;
671
+ if (!result.success && !partialFailure) {
672
+ throw new Error(result.error ?? 'Generation failed');
673
+ }
674
+ if (result.background) {
675
+ const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
676
+ return backgroundSubmitted(`${imageModel} image generation`, ids, {
677
+ model: imageModel,
678
+ costKey,
679
+ projectId: input.projectId,
680
+ cost_cents: totalCents,
681
+ cost_dollars: (totalCents / 100).toFixed(2),
682
+ });
683
+ }
684
+ const assetList = result.assets
685
+ ?? (result.asset ? [result.asset] : []);
686
+ const images = [];
687
+ for (const asset of assetList) {
688
+ const id = asset.id;
689
+ if (!id)
690
+ continue;
691
+ try {
692
+ const img = await desktop.get('/agent/assets/image', { id });
693
+ images.push({ data: img.data, mimeType: img.mimeType });
694
+ }
695
+ catch {
696
+ // Vision payload is best-effort; skip if the disk read fails.
697
+ }
698
+ }
699
+ const requestedCount = input.count ?? 1;
700
+ return {
701
+ text: partialFailure
702
+ ? `Partial result: ${assetList.length} of ${requestedCount} image(s) saved into project ${input.projectId} ` +
703
+ `(error on the rest: ${result.error ?? 'unknown error'}). ` +
704
+ `The saved assets are in data.assets — re-generate only the missing count, don't redo the whole batch. ` +
705
+ `Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`
706
+ : `Generated ${assetList.length} image(s) into project ${input.projectId} ` +
707
+ `for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
708
+ `Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
709
+ images,
710
+ data: {
711
+ model: imageModel,
712
+ costKey,
713
+ projectId: input.projectId,
714
+ aspectRatio: input.aspectRatio,
715
+ resolution,
716
+ cost_cents: totalCents,
717
+ cost_dollars: (totalCents / 100).toFixed(2),
718
+ assets: assetList,
719
+ generationIds: result.generationIds ?? (result.generationId ? [result.generationId] : []),
720
+ ...(partialFailure
721
+ ? { partial: true, error: result.error ?? 'unknown error', requested_count: requestedCount }
722
+ : {}),
723
+ },
724
+ };
377
725
  }
378
- // Generation itself routes through /proxy/generate, which is the
379
- // existing slates-api credit-aware path. The desktop app handles the
380
- // proxy interaction here we use the cloud client to call /proxy
381
- // ourselves so the agent can run without the desktop server. The
382
- // proxy returns base64 image bytes inline (Google streaming path) or
383
- // an asynced jobId we'd poll. For v1 we go through nano-banana on fal,
384
- // which returns a CDN URL; we fetch the URL and base64-encode here.
726
+ // /proxy/generate kicks off a credit-aware job and returns a jobId
727
+ // for fal/Veo (async providers). We poll /proxy/jobs/{jobId} until
728
+ // the status is `completed` or `failed`, then fetch each image URL
729
+ // and inline as base64 so the calling LLM sees the pixels.
730
+ // The fal endpoint differs by mode: bare model id for text-to-image,
731
+ // `/edit` when reference image URLs are provided. Sending text-to-image
732
+ // to `/edit` 422s on the missing `image_urls` field.
733
+ const hasReferenceImages = !!input.referenceImageUrls && input.referenceImageUrls.length > 0;
734
+ const endpoint = hasReferenceImages
735
+ ? 'fal-ai/nano-banana-2/edit'
736
+ : 'fal-ai/nano-banana-2';
385
737
  const proxyResult = await cloud.post('/proxy/generate', {
386
738
  provider: 'fal',
387
739
  type: 'image',
388
- model,
389
- endpoint: 'fal-ai/nano-banana-2/edit',
740
+ model: costKey,
741
+ endpoint,
390
742
  params: {
391
743
  prompt: input.prompt,
392
744
  aspect_ratio: input.aspectRatio ?? '1:1',
393
745
  num_images: input.count ?? 1,
394
- ...(input.referenceImageUrls && input.referenceImageUrls.length > 0
746
+ ...(hasReferenceImages
395
747
  ? { image_urls: input.referenceImageUrls }
396
748
  : {}),
397
749
  },
398
750
  });
399
- if (!proxyResult.success || !proxyResult.data) {
751
+ if (!proxyResult.success) {
400
752
  throw new Error(proxyResult.error ?? 'Generation failed');
401
753
  }
402
- const urls = (proxyResult.data.images ?? []).map((i) => i.url).filter((u) => !!u);
754
+ let imagesPayload = proxyResult.data?.images;
755
+ if (!imagesPayload && proxyResult.jobId) {
756
+ const completed = await pollProxyJob(cloud, proxyResult.jobId);
757
+ imagesPayload = completed.images;
758
+ }
759
+ const urls = (imagesPayload ?? [])
760
+ .map((i) => i.url)
761
+ .filter((u) => !!u);
762
+ if (urls.length === 0) {
763
+ throw new Error('Generation completed but returned no image URLs.');
764
+ }
403
765
  const images = [];
404
766
  for (const url of urls) {
405
767
  const r = await fetch(url);
@@ -408,9 +770,1090 @@ export const generateImage = {
408
770
  images.push({ data: buf.toString('base64'), mimeType: mt });
409
771
  }
410
772
  return {
411
- text: `Generated ${urls.length} image(s) for prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
773
+ text: `Generated ${urls.length} image(s) for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
774
+ `Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
775
+ images,
776
+ data: {
777
+ urls,
778
+ model: imageModel,
779
+ costKey,
780
+ aspectRatio: input.aspectRatio,
781
+ resolution,
782
+ cost_cents: totalCents,
783
+ cost_dollars: (totalCents / 100).toFixed(2),
784
+ },
785
+ };
786
+ },
787
+ };
788
+ async function pollProxyJob(cloud, jobId, options = {}) {
789
+ const intervalMs = options.intervalMs ?? 3000;
790
+ const timeoutMs = options.timeoutMs ?? 5 * 60 * 1000;
791
+ const start = Date.now();
792
+ while (Date.now() - start < timeoutMs) {
793
+ await new Promise((r) => setTimeout(r, intervalMs));
794
+ let status;
795
+ try {
796
+ status = await cloud.get(`/proxy/jobs/${jobId}`);
797
+ }
798
+ catch {
799
+ continue;
800
+ }
801
+ if (status.status === 'completed')
802
+ return status.data ?? {};
803
+ if (status.status === 'failed') {
804
+ throw new Error(status.error ?? 'Generation failed');
805
+ }
806
+ }
807
+ throw new Error(`Generation timed out after ${Math.round(timeoutMs / 1000)}s.`);
808
+ }
809
+ // ── Edit image ──────────────────────────────────────────────────
810
+ export const editImage = {
811
+ id: 'slates_edit_image',
812
+ description: 'Surgically edit an existing image asset with a text instruction (e.g. \'remove the lamppost\', \'make the jacket red\') instead of regenerating from scratch — use when ~90% of the image is already right. The edited result is saved as a NEW asset in the project (prompt prefixed \'[Edit]\'); the source is untouched. Default model nano-banana-2 (only model that also accepts referenceAssetIds); flux-2-max / seedream-5-lite use their own edit endpoints and ignore references. Before first use call slates_get_prompting_guide with topic \'slates-edit-and-iterate\'.',
813
+ input: z.object({
814
+ projectId: z.string().uuid(),
815
+ sourceAssetId: z.string().uuid().describe('Image asset to edit. Must already exist in the project.'),
816
+ prompt: z.string().min(1).max(4000).describe('The edit instruction — describe the change, not the whole image.'),
817
+ editModel: z.enum(['nano-banana-2', 'flux-2-max', 'seedream-5-lite']).optional(),
818
+ referenceAssetIds: z.array(z.string().uuid()).max(13).optional().describe('nano-banana-2 only: extra reference images.'),
819
+ resolution: z.enum(['1k', '2k', '4k']).optional(),
820
+ aspectRatio: z.string().optional(),
821
+ confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate.'),
822
+ background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
823
+ }),
824
+ async run(input, ctx) {
825
+ const desktop = ctx.desktop();
826
+ await desktop.requireCapability('edit-image', 'image editing');
827
+ if (input.background) {
828
+ await desktop.requireCapability('background-generation', 'background generation');
829
+ }
830
+ const editModel = input.editModel ?? 'nano-banana-2';
831
+ const resolution = input.resolution ?? '2k';
832
+ // NB2 edits charge the normal per-resolution NB2 key; FLUX / Seedream
833
+ // route to their dedicated edit endpoints, priced under '-edit' keys.
834
+ const costKey = editModel === 'nano-banana-2'
835
+ ? imageCostKey('nano-banana-2', resolution)
836
+ : `${imageCostKey(editModel, resolution)}-edit`;
837
+ const cloud = ctx.cloud();
838
+ const registry = await cloud.get('/api/agent/models');
839
+ const entry = registry.models.find((m) => m.model === costKey);
840
+ if (!entry)
841
+ throw new Error(`Model not in registry: ${costKey}`);
842
+ const totalCents = entry.cost_cents;
843
+ if (totalCents > 50 && !input.confirm) {
844
+ const sourceRef = await lookupAssetRef(desktop, input.sourceAssetId);
845
+ return ok({
846
+ requires_confirm: true,
847
+ variant: costKey,
848
+ estimated_cents: totalCents,
849
+ estimated_dollars: (totalCents / 100).toFixed(2),
850
+ source_ref: sourceRef,
851
+ message: `Cost: $${(totalCents / 100).toFixed(2)} to edit ${sourceRef} with ${editModel} (${costKey}). ` +
852
+ `Re-call with confirm=true after the user explicitly OKs the spend. ` +
853
+ `When discussing with the user, refer to the source by its code (matches the gallery badge).`,
854
+ });
855
+ }
856
+ const result = await desktop.post('/agent/generation/edit-image', {
857
+ projectId: input.projectId,
858
+ sourceAssetId: input.sourceAssetId,
859
+ prompt: input.prompt,
860
+ editModel,
861
+ referenceAssetIds: input.referenceAssetIds,
862
+ resolution,
863
+ aspectRatio: input.aspectRatio,
864
+ background: input.background,
865
+ });
866
+ if (!result.success)
867
+ throw new Error(result.error ?? 'Image edit failed');
868
+ if (result.background) {
869
+ const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
870
+ return backgroundSubmitted(`${editModel} image edit`, ids, {
871
+ editModel,
872
+ variant: costKey,
873
+ projectId: input.projectId,
874
+ sourceAssetId: input.sourceAssetId,
875
+ cost_cents: totalCents,
876
+ cost_dollars: (totalCents / 100).toFixed(2),
877
+ });
878
+ }
879
+ // Inline the edited result so the LLM sees whether the surgery landed —
880
+ // same best-effort pattern as slates_generate_image.
881
+ const images = [];
882
+ const newAssetId = result.asset?.id;
883
+ if (newAssetId) {
884
+ try {
885
+ const img = await desktop.get('/agent/assets/image', { id: newAssetId });
886
+ images.push({ data: img.data, mimeType: img.mimeType });
887
+ }
888
+ catch {
889
+ // Vision payload is best-effort; skip if the disk read fails.
890
+ }
891
+ }
892
+ return {
893
+ text: `Edited image saved as a new asset in project ${input.projectId} ` +
894
+ `for $${(totalCents / 100).toFixed(2)} (${totalCents}¢) via ${editModel}. ` +
895
+ `Edit: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
412
896
  images,
413
- data: { urls, model },
897
+ data: {
898
+ editModel,
899
+ variant: costKey,
900
+ projectId: input.projectId,
901
+ sourceAssetId: input.sourceAssetId,
902
+ resolution,
903
+ cost_cents: totalCents,
904
+ cost_dollars: (totalCents / 100).toFixed(2),
905
+ asset: result.asset,
906
+ generationId: result.generationId,
907
+ },
908
+ };
909
+ },
910
+ };
911
+ // ── Generate video ──────────────────────────────────────────────
912
+ const VIDEO_MODELS = [
913
+ 'kling-v3.0-std',
914
+ 'kling-v3.0-pro',
915
+ 'kling-v3.0-omni',
916
+ 'veo-3.1-fast',
917
+ 'veo-3.1-standard',
918
+ 'seedance-2-fast',
919
+ 'seedance-2-std',
920
+ ];
921
+ // Model → registry cost-key. Each provider's keys ship with their own
922
+ // shape (verified against /api/agent/models):
923
+ // Kling: kling-v3-{standard|pro|omni}-{N}s — note the user-facing
924
+ // model id `kling-v3.0-std` maps to registry key `kling-v3-standard`.
925
+ // Veo: veo-3.1-{fast|standard}[-4k]-{N}s[-audio]
926
+ // Seedance: seedance-2-{fast|std}[-priority]-{N}s. Economy (PiAPI) and
927
+ // Priority (fal.ai) are DIFFERENT registry keys at DIFFERENT prices —
928
+ // priority is ~2.2x economy. The cost key MUST encode the tier or the
929
+ // pre-flight quote understates a priority gen (the desktop charges the
930
+ // priority key regardless of what we quoted).
931
+ const KLING_TIER_MAP = {
932
+ 'kling-v3.0-std': 'kling-v3-standard',
933
+ 'kling-v3.0-pro': 'kling-v3-pro',
934
+ 'kling-v3.0-omni': 'kling-v3-omni',
935
+ };
936
+ function videoCostKey(input) {
937
+ if (input.model.startsWith('seedance')) {
938
+ // Mirrors seedanceCreditKey() in slate/src/shared/pricing.ts.
939
+ return input.seedanceSpeed === 'priority'
940
+ ? `${input.model}-priority-${input.duration}s`
941
+ : `${input.model}-${input.duration}s`;
942
+ }
943
+ if (input.model.startsWith('veo')) {
944
+ const is4k = input.videoResolution === '4k';
945
+ const audio = input.sound !== false; // default audio on for Veo
946
+ const parts = [input.model];
947
+ if (is4k)
948
+ parts.push('4k');
949
+ parts.push(`${input.duration}s`);
950
+ if (audio)
951
+ parts.push('audio');
952
+ return parts.join('-');
953
+ }
954
+ if (input.model.startsWith('kling-v3.0')) {
955
+ const tier = KLING_TIER_MAP[input.model] ?? input.model;
956
+ return `${tier}-${input.duration}s`;
957
+ }
958
+ throw new Error(`Unknown video model: ${input.model}`);
959
+ }
960
+ // Maps a video model id to its bundled prompting skill (frontmatter `name:`),
961
+ // so guidance text points at a skill that actually exists. Deriving the name
962
+ // via model.split('-')[0] produced 'slates-prompting-kling' / '...-veo', which
963
+ // match no file — only seedance happened to line up.
964
+ function promptingSkillFor(model) {
965
+ if (model.startsWith('kling'))
966
+ return 'slates-prompting-kling-v3';
967
+ if (model.startsWith('veo'))
968
+ return 'slates-prompting-veo-3';
969
+ if (model.startsWith('seedance'))
970
+ return 'slates-prompting-seedance';
971
+ return 'slates-cost-discipline';
972
+ }
973
+ export const generateVideo = {
974
+ id: 'slates_generate_video',
975
+ description: 'Generate video via Slates credits. REQUIRED before calling: read the slates-cost-discipline skill plus the per-model prompting skill (slates-prompting-seedance / slates-prompting-kling-v3 / slates-prompting-veo-3) — video models prompt very differently. projectId is REQUIRED for UI integration (the user sees a progress card and the asset lands in the project — without it the call fails). aspectRatio + duration are required (server returns requires_clarification when missing). Cost > $0.50 returns requires_confirm — pass confirm=true after explicit user OK. Veo locks to 16:9 and to 4/6/8s durations (4K only at 8s). Image-to-video via firstFrameAssetId. Frames-to-video via firstFrameAssetId + lastFrameAssetId (Veo / Seedance only). Ingredients via ingredientAssetIds (Kling Omni / Seedance). No skill files installed? Call slates_get_prompting_guide with the per-model guide (\'slates-prompting-veo-3\' / \'slates-prompting-kling-v3\' / \'slates-prompting-seedance\') and \'slates-cost-discipline\' before first use.',
976
+ input: z.object({
977
+ prompt: z.string().min(1).max(4000),
978
+ model: z.enum(VIDEO_MODELS).describe('Pick deliberately by capability AND cost. Kling V3.0 std = cheapest (no audio); pro = mid; omni = multi-char dialogue + audio. Veo 3.1 = top quality, locks 16:9, audio; fast vs standard. Seedance 2 = ByteDance, audio included, supports first+last frame; pick economy vs priority via seedanceSpeed. For exact per-call credit cost, call slates_estimate_generation_cost or slates_list_available_models — never quote prices from memory (they change).'),
979
+ projectId: z.string().uuid().optional().describe('Save into this Slates project. Strongly recommended — the desktop UI shows a progress card live and the asset appears when complete.'),
980
+ aspectRatio: z.enum(['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21', '4:5', '5:4', '2:3', '3:2']).optional().describe('Veo locks to 16:9 — passing anything else will be ignored or fail. Kling/Seedance support all.'),
981
+ duration: z.number().int().min(4).max(15).optional().describe('Seconds. Kling: 5-15. Veo: 4, 6, or 8 only (4K only at 8s). Seedance: 4-15. Default 5 if omitted but always be explicit (cost scales linearly).'),
982
+ videoResolution: z.enum(['720p', '1080p', '4k']).optional().describe('Veo only. 720p / 1080p same price. 4k more expensive.'),
983
+ seedanceSpeed: z.enum(['economy', 'priority']).optional().describe('Seedance only. Economy via PiAPI (cheaper, slower). Priority via fal.ai (faster).'),
984
+ firstFrameAssetId: z.string().uuid().optional().describe('Asset id from the project — used as the starting frame for image-to-video. Must already exist in the project.'),
985
+ lastFrameAssetId: z.string().uuid().optional().describe('Asset id from the project — used as the ending frame. Veo and Seedance only. Pairs with firstFrameAssetId for guided transitions.'),
986
+ ingredientAssetIds: z.array(z.string().uuid()).max(9).optional().describe('Asset ids used as visual reference / ingredients for Kling Omni or Seedance. Up to 9 (Seedance) or 4 (Kling).'),
987
+ sound: z.boolean().optional().describe('Kling Omni / Veo / Seedance: enable audio generation. Default true.'),
988
+ audioLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('Kling Omni only — language for dialogue.'),
989
+ generateMusic: z.boolean().optional().describe('Kling Omni only — auto-generate background music.'),
990
+ negativePrompt: z.string().optional(),
991
+ background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
992
+ confirm: z.boolean().optional().describe('Set true after explicit user OK to bypass the >$0.50 cost confirm gate (which fires for almost every video gen since they\'re expensive).'),
993
+ }),
994
+ async run(input, ctx) {
995
+ // projectId is required for video — without it there's no UI feedback,
996
+ // no asset to reference later, and a failed gen leaves the user with
997
+ // nothing. The MCP-only headless path that exists for image gen is
998
+ // not reasonable for video given the cost.
999
+ if (!input.projectId) {
1000
+ return ok({
1001
+ requires_clarification: true,
1002
+ missing: ['projectId'],
1003
+ message: 'projectId is required for video generation. Use slates_list_projects to find one or slates_create_project to make a new one. Video gens cost $0.50-$2.00+ per call — they need to land in a project so the user sees the progress card and the result.',
1004
+ });
1005
+ }
1006
+ if (!input.aspectRatio || !input.duration) {
1007
+ const missing = [];
1008
+ if (!input.aspectRatio)
1009
+ missing.push('aspectRatio');
1010
+ if (!input.duration)
1011
+ missing.push('duration');
1012
+ return ok({
1013
+ requires_clarification: true,
1014
+ missing,
1015
+ message: `Missing required field(s): ${missing.join(', ')}. ` +
1016
+ `Read the slates-cost-discipline + ${promptingSkillFor(input.model)} skills, ` +
1017
+ `or ask the user. Veo locks to 16:9. Kling/Seedance support 1:1 16:9 9:16 4:3 3:4 21:9. ` +
1018
+ `Duration: Kling 5-15s, Veo 4/6/8s (4K only at 8s), Seedance 4-15s. Cost scales linearly with duration.`,
1019
+ });
1020
+ }
1021
+ // Veo exists only at discrete durations 4/6/8s, and 4K only at 8s.
1022
+ // Validate up front so the agent gets an actionable message instead of
1023
+ // a generic "Model variant not in registry" throw from the cost lookup.
1024
+ if (input.model.startsWith('veo')) {
1025
+ if (![4, 6, 8].includes(input.duration)) {
1026
+ return ok({
1027
+ requires_clarification: true,
1028
+ missing: ['duration'],
1029
+ message: `Veo 3.1 supports only 4s, 6s, or 8s (you passed ${input.duration}s). Pick one of those.`,
1030
+ });
1031
+ }
1032
+ if (input.videoResolution === '4k' && input.duration !== 8) {
1033
+ return ok({
1034
+ requires_clarification: true,
1035
+ missing: ['duration'],
1036
+ message: `Veo 3.1 4K renders only at 8s (you passed ${input.duration}s at 4k). Use duration=8 for 4K, or drop to 720p/1080p for 4s/6s.`,
1037
+ });
1038
+ }
1039
+ }
1040
+ const cloud = ctx.cloud();
1041
+ const registry = await cloud.get('/api/agent/models');
1042
+ const costKey = videoCostKey({
1043
+ model: input.model,
1044
+ duration: input.duration,
1045
+ videoResolution: input.videoResolution,
1046
+ sound: input.sound,
1047
+ seedanceSpeed: input.seedanceSpeed,
1048
+ });
1049
+ const entry = registry.models.find((m) => m.model === costKey);
1050
+ if (!entry) {
1051
+ throw new Error(`Model variant not in registry: ${costKey}. ` +
1052
+ `Available video models: ${registry.models.filter((m) => m.model.startsWith('kling') || m.model.startsWith('veo') || m.model.startsWith('seedance')).map((m) => m.model).slice(0, 20).join(', ')}`);
1053
+ }
1054
+ const totalCents = entry.cost_cents;
1055
+ // Pre-flight confirm gate. Fires when:
1056
+ // (a) cost > $0.50 (the cost gate), OR
1057
+ // (b) any reference assets are involved (the look-first gate)
1058
+ // When references are present, the response inlines them as image
1059
+ // content blocks (and video keyframes for video refs) so the LLM
1060
+ // literally sees what it's about to use before committing spend.
1061
+ // The text steers the LLM to discuss the refs with the user by their
1062
+ // code+label so the conversation maps onto the gallery badges.
1063
+ const referenceRefs = [];
1064
+ if (input.firstFrameAssetId) {
1065
+ referenceRefs.push({ id: input.firstFrameAssetId, type: 'image', role: 'first frame' });
1066
+ }
1067
+ if (input.lastFrameAssetId) {
1068
+ referenceRefs.push({ id: input.lastFrameAssetId, type: 'image', role: 'last frame' });
1069
+ }
1070
+ for (const id of input.ingredientAssetIds ?? []) {
1071
+ referenceRefs.push({ id, type: 'image', role: 'ingredient' });
1072
+ }
1073
+ const hasReferences = referenceRefs.length > 0;
1074
+ if ((totalCents > 50 || hasReferences) && !input.confirm) {
1075
+ const previews = hasReferences ? await previewAssets(ctx, referenceRefs) : [];
1076
+ const refLines = previews.map((p) => ` - ${p.role}: ${p.ref}`).join('\n');
1077
+ const refSummary = hasReferences
1078
+ ? `\n\nReferences attached above (in order — first frame, last frame, then ingredients):\n${refLines}\n\nReview them against your prompt. If the references suggest a different motion / framing / ` +
1079
+ `style than the current prompt captures, REVISE the prompt before confirming. When you talk to the user about this gen, refer to each reference by its code (e.g. "${previews[0]?.ref ?? 'IMG-A?'}") — they'll see the matching badge in the Slates gallery.`
1080
+ : '';
1081
+ const refImages = previews.flatMap((p) => p.images);
1082
+ return {
1083
+ text: `Pre-flight for ${input.duration}s ${input.model} (${costKey}): ` +
1084
+ `$${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
1085
+ refSummary +
1086
+ `\n\nWhen ready, re-call slates_generate_video with confirm=true and the (possibly revised) prompt.`,
1087
+ images: refImages,
1088
+ data: {
1089
+ requires_confirm: true,
1090
+ model: input.model,
1091
+ variant: costKey,
1092
+ estimated_cents: totalCents,
1093
+ estimated_dollars: (totalCents / 100).toFixed(2),
1094
+ references: previews.map((p) => ({
1095
+ role: p.role,
1096
+ ref: p.ref,
1097
+ asset_id: p.meta.asset_id,
1098
+ code: p.meta.code,
1099
+ label: p.meta.label,
1100
+ type: p.meta.type,
1101
+ frame_count: p.meta.frame_count,
1102
+ })),
1103
+ },
1104
+ };
1105
+ }
1106
+ const desktop = ctx.desktop();
1107
+ if (input.background) {
1108
+ await desktop.requireCapability('background-generation', 'background generation');
1109
+ }
1110
+ const result = await desktop.post('/agent/generation/video', {
1111
+ projectId: input.projectId,
1112
+ model: input.model,
1113
+ prompt: input.prompt,
1114
+ aspectRatio: input.aspectRatio,
1115
+ duration: input.duration,
1116
+ videoResolution: input.videoResolution,
1117
+ seedanceSpeed: input.seedanceSpeed,
1118
+ firstFrameAssetId: input.firstFrameAssetId,
1119
+ lastFrameAssetId: input.lastFrameAssetId,
1120
+ ingredientAssetIds: input.ingredientAssetIds ?? [],
1121
+ sound: input.sound,
1122
+ audioLanguage: input.audioLanguage,
1123
+ generateMusic: input.generateMusic,
1124
+ negativePrompt: input.negativePrompt,
1125
+ background: input.background,
1126
+ });
1127
+ if (!result.success) {
1128
+ throw new Error(result.error ?? 'Generation failed');
1129
+ }
1130
+ if (result.background) {
1131
+ const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
1132
+ return backgroundSubmitted(`${input.duration}s ${input.model} video generation`, ids, {
1133
+ model: input.model,
1134
+ variant: costKey,
1135
+ projectId: input.projectId,
1136
+ cost_cents: totalCents,
1137
+ cost_dollars: (totalCents / 100).toFixed(2),
1138
+ });
1139
+ }
1140
+ return {
1141
+ text: `Generated ${input.duration}s ${input.model} video into project ${input.projectId} ` +
1142
+ `for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
1143
+ `Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"`,
1144
+ data: {
1145
+ model: input.model,
1146
+ variant: costKey,
1147
+ projectId: input.projectId,
1148
+ aspectRatio: input.aspectRatio,
1149
+ duration: input.duration,
1150
+ cost_cents: totalCents,
1151
+ cost_dollars: (totalCents / 100).toFixed(2),
1152
+ asset: result.asset,
1153
+ generationId: result.generationId,
1154
+ },
1155
+ };
1156
+ },
1157
+ };
1158
+ // ── Generate lip-sync ───────────────────────────────────────────
1159
+ export const generateLipSync = {
1160
+ id: 'slates_generate_lip_sync',
1161
+ description: 'Lip-sync a still image (avatar) or a video clip to audio via Kling. REQUIRED before calling: read the slates-cost-discipline + slates-prompting-lip-sync skills. projectId is REQUIRED — the source asset must already exist in the project. Two flows: (1) sourceType=video → lip-syncs an existing talking-head clip to new audio (~$0.11 / 5s, no confirm gate); (2) sourceType=image → animates a still portrait into a talking avatar (avatar-standard ~$0.42 / 5s; avatar-pro ~$0.86 / 5s, hits the >$0.50 confirm gate). Audio comes from either TTS (pass ttsText) or an uploaded file (pass audioFilePath). Always 5 seconds — Kling lip-sync does not support other durations. No skill files installed? Call slates_get_prompting_guide with \'slates-prompting-lip-sync\' (and \'slates-cost-discipline\') before first use.',
1162
+ input: z.object({
1163
+ projectId: z.string().uuid().describe('Slates project the source asset lives in. The new lip-synced video lands here.'),
1164
+ sourceAssetId: z.string().uuid().describe('Asset id of the still image (avatar flow) or video clip (lip-sync flow). Must already exist in the project — use slates_upload_reference_image or slates_generate_image / slates_generate_video first if needed.'),
1165
+ sourceType: z.enum(['image', 'video']).describe('"image" = animate a still portrait (avatar). "video" = re-sync an existing talking-head clip. Determines pricing — be deliberate.'),
1166
+ audioMethod: z.enum(['tts', 'upload']).describe('"tts" = generate speech from ttsText. "upload" = use the file at audioFilePath (absolute path on the user\'s machine).'),
1167
+ ttsText: z.string().min(1).max(2000).optional().describe('Required when audioMethod=tts. The exact words the avatar/clip will speak.'),
1168
+ ttsVoice: z.string().optional().describe('Kling voice id (e.g. "oversea_male1"). See slates-prompting-lip-sync skill for the voice catalog.'),
1169
+ ttsLanguage: z.enum(['EN', 'ZH', 'JA', 'KO', 'ES']).optional().describe('TTS language. Default EN.'),
1170
+ ttsSpeed: z.number().min(0.5).max(2).optional().describe('TTS speech rate. Default 1.0. Range 0.5-2.0.'),
1171
+ audioFilePath: z.string().optional().describe('Required when audioMethod=upload. Absolute path to the audio file on the user\'s machine (mp3, wav, m4a).'),
1172
+ avatarModel: z.enum(['avatar-standard', 'avatar-pro']).optional().describe('Image-source only. avatar-standard ($0.42/5s) for general use. avatar-pro ($0.86/5s) for sharper face fidelity. Ignored when sourceType=video.'),
1173
+ background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
1174
+ confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 cost confirm gate. Required for avatar-pro.'),
1175
+ }),
1176
+ async run(input, ctx) {
1177
+ if (input.audioMethod === 'tts' && !input.ttsText) {
1178
+ return ok({
1179
+ requires_clarification: true,
1180
+ missing: ['ttsText'],
1181
+ message: 'audioMethod=tts requires ttsText. Pass the exact words the avatar/clip will speak.',
1182
+ });
1183
+ }
1184
+ if (input.audioMethod === 'upload' && !input.audioFilePath) {
1185
+ return ok({
1186
+ requires_clarification: true,
1187
+ missing: ['audioFilePath'],
1188
+ message: 'audioMethod=upload requires audioFilePath. Pass an absolute path to the audio file on the user\'s machine.',
1189
+ });
1190
+ }
1191
+ let costKey;
1192
+ if (input.sourceType === 'video') {
1193
+ costKey = 'kling-lip-sync-video-5s';
1194
+ }
1195
+ else {
1196
+ costKey = input.avatarModel === 'avatar-pro'
1197
+ ? 'kling-lip-sync-avatar-pro-5s'
1198
+ : 'kling-lip-sync-avatar-5s';
1199
+ }
1200
+ const cloud = ctx.cloud();
1201
+ const registry = await cloud.get('/api/agent/models');
1202
+ const entry = registry.models.find((m) => m.model === costKey);
1203
+ if (!entry)
1204
+ throw new Error(`Model variant not in registry: ${costKey}`);
1205
+ const totalCents = entry.cost_cents;
1206
+ // Cost confirm gate. Lip-sync is mechanical — the model re-syncs the
1207
+ // user-chosen source to the user-chosen audio. The agent doesn't
1208
+ // write a prompt that depends on what the source looks like, so we
1209
+ // skip the inline preview and just announce the source code in text.
1210
+ if (totalCents > 50 && !input.confirm) {
1211
+ const sourceRef = await lookupAssetRef(ctx.desktop(), input.sourceAssetId);
1212
+ const audioPreview = input.audioMethod === 'tts'
1213
+ ? `Audio: TTS — "${(input.ttsText ?? '').slice(0, 120)}"`
1214
+ : `Audio: ${input.audioFilePath}`;
1215
+ return ok({
1216
+ requires_confirm: true,
1217
+ variant: costKey,
1218
+ estimated_cents: totalCents,
1219
+ estimated_dollars: (totalCents / 100).toFixed(2),
1220
+ source_ref: sourceRef,
1221
+ message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${costKey}. ` +
1222
+ `Source: ${sourceRef}. ${audioPreview}. ` +
1223
+ `Re-call with confirm=true after the user explicitly OKs the spend. ` +
1224
+ `When discussing with the user, refer to the source by its code (matches the gallery badge).`,
1225
+ });
1226
+ }
1227
+ const desktop = ctx.desktop();
1228
+ if (input.background) {
1229
+ await desktop.requireCapability('background-generation', 'background generation');
1230
+ }
1231
+ const result = await desktop.post('/agent/generation/lip-sync', {
1232
+ projectId: input.projectId,
1233
+ sourceAssetId: input.sourceAssetId,
1234
+ audioMethod: input.audioMethod,
1235
+ ttsText: input.ttsText,
1236
+ ttsVoice: input.ttsVoice,
1237
+ ttsLanguage: input.ttsLanguage,
1238
+ ttsSpeed: input.ttsSpeed,
1239
+ audioFilePath: input.audioFilePath,
1240
+ avatarModel: input.avatarModel,
1241
+ estimatedCost: totalCents,
1242
+ background: input.background,
1243
+ });
1244
+ if (!result.success)
1245
+ throw new Error(result.error ?? 'Lip-sync generation failed');
1246
+ if (result.background) {
1247
+ const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
1248
+ return backgroundSubmitted(`5s lip-sync (${costKey})`, ids, {
1249
+ variant: costKey,
1250
+ projectId: input.projectId,
1251
+ sourceAssetId: input.sourceAssetId,
1252
+ cost_cents: totalCents,
1253
+ cost_dollars: (totalCents / 100).toFixed(2),
1254
+ });
1255
+ }
1256
+ return {
1257
+ text: `Generated 5s lip-sync (${costKey}) into project ${input.projectId} ` +
1258
+ `for $${(totalCents / 100).toFixed(2)} (${totalCents}¢). ` +
1259
+ (input.audioMethod === 'tts'
1260
+ ? `Spoken: "${(input.ttsText ?? '').slice(0, 60)}${(input.ttsText ?? '').length > 60 ? '...' : ''}"`
1261
+ : `Audio: ${input.audioFilePath}`),
1262
+ data: {
1263
+ variant: costKey,
1264
+ projectId: input.projectId,
1265
+ sourceType: input.sourceType,
1266
+ sourceAssetId: input.sourceAssetId,
1267
+ cost_cents: totalCents,
1268
+ cost_dollars: (totalCents / 100).toFixed(2),
1269
+ asset: result.asset,
1270
+ generationId: result.generationId,
1271
+ },
1272
+ };
1273
+ },
1274
+ };
1275
+ // ── Generate motion transfer ────────────────────────────────────
1276
+ export const generateMotionTransfer = {
1277
+ id: 'slates_generate_motion_transfer',
1278
+ description: 'Transfer the motion from a reference video onto a target image character via Kling Motion Control. REQUIRED before calling: read the slates-cost-discipline + slates-prompting-motion-transfer skills. projectId is REQUIRED — both source video and target image must already exist as assets in the project. Two tiers: kling-mc-std ($0.95 / 5s) and kling-mc-pro ($1.26 / 5s) — both hit the >$0.50 confirm gate. Always 5 seconds. No skill files installed? Call slates_get_prompting_guide with \'slates-prompting-motion-transfer\' (and \'slates-cost-discipline\') before first use.',
1279
+ input: z.object({
1280
+ projectId: z.string().uuid().describe('Slates project. Both source and target assets must live here.'),
1281
+ sourceVideoAssetId: z.string().uuid().describe('Asset id of the reference video — its motion will be retargeted onto the target image. Must already exist in the project.'),
1282
+ targetImageAssetId: z.string().uuid().describe('Asset id of the target image (the character that will perform the motion). Must already exist in the project.'),
1283
+ motionModel: z.enum(['kling-mc-std', 'kling-mc-pro']).optional().describe('std ($0.95) for general motion. pro ($1.26) for cleaner anatomy + identity preservation. Default pro — pick std deliberately for cost savings.'),
1284
+ characterOrientation: z.enum(['video', 'image']).optional().describe('"video" = use the source video\'s framing. "image" = use the target image\'s framing. Default video.'),
1285
+ prompt: z.string().optional().describe('Optional refinement prompt. Read slates-prompting-motion-transfer for guidance.'),
1286
+ background: z.boolean().optional().describe(BACKGROUND_DESCRIBE),
1287
+ confirm: z.boolean().optional().describe('Set true to bypass the >$0.50 confirm gate. Required — both tiers exceed.'),
1288
+ }),
1289
+ async run(input, ctx) {
1290
+ const motionModel = input.motionModel ?? 'kling-mc-pro';
1291
+ const costKey = motionModel === 'kling-mc-std' ? 'kling-mc-std-5s' : 'kling-mc-pro-5s';
1292
+ const cloud = ctx.cloud();
1293
+ const registry = await cloud.get('/api/agent/models');
1294
+ const entry = registry.models.find((m) => m.model === costKey);
1295
+ if (!entry)
1296
+ throw new Error(`Model variant not in registry: ${costKey}`);
1297
+ const totalCents = entry.cost_cents;
1298
+ // Cost confirm gate. Motion transfer is mechanical — the model
1299
+ // applies source motion to target image deterministically. We don't
1300
+ // burn tokens previewing assets the user already chose; codes in the
1301
+ // text are enough to keep the chat unambiguous.
1302
+ if (totalCents > 50 && !input.confirm) {
1303
+ const desktop = ctx.desktop();
1304
+ const [source, target] = await Promise.all([
1305
+ lookupAssetRef(desktop, input.sourceVideoAssetId),
1306
+ lookupAssetRef(desktop, input.targetImageAssetId),
1307
+ ]);
1308
+ return ok({
1309
+ requires_confirm: true,
1310
+ variant: costKey,
1311
+ estimated_cents: totalCents,
1312
+ estimated_dollars: (totalCents / 100).toFixed(2),
1313
+ source_ref: source,
1314
+ target_ref: target,
1315
+ message: `Cost: $${(totalCents / 100).toFixed(2)} for 5s ${motionModel}. ` +
1316
+ `Transferring motion from ${source} onto ${target}. ` +
1317
+ `Re-call with confirm=true after the user explicitly OKs the spend, or pick kling-mc-std to save $0.31. ` +
1318
+ `When discussing with the user, refer to the assets by those codes — they'll match the gallery badges.`,
1319
+ });
1320
+ }
1321
+ const desktop = ctx.desktop();
1322
+ if (input.background) {
1323
+ await desktop.requireCapability('background-generation', 'background generation');
1324
+ }
1325
+ const result = await desktop.post('/agent/generation/motion-transfer', {
1326
+ projectId: input.projectId,
1327
+ sourceVideoAssetId: input.sourceVideoAssetId,
1328
+ targetImageAssetId: input.targetImageAssetId,
1329
+ motionModel,
1330
+ characterOrientation: input.characterOrientation ?? 'video',
1331
+ prompt: input.prompt,
1332
+ estimatedCost: totalCents,
1333
+ background: input.background,
1334
+ });
1335
+ if (!result.success)
1336
+ throw new Error(result.error ?? 'Motion transfer generation failed');
1337
+ if (result.background) {
1338
+ const ids = result.generationIds ?? (result.generationId ? [result.generationId] : []);
1339
+ return backgroundSubmitted(`5s motion transfer (${motionModel})`, ids, {
1340
+ variant: costKey,
1341
+ motionModel,
1342
+ projectId: input.projectId,
1343
+ sourceVideoAssetId: input.sourceVideoAssetId,
1344
+ targetImageAssetId: input.targetImageAssetId,
1345
+ cost_cents: totalCents,
1346
+ cost_dollars: (totalCents / 100).toFixed(2),
1347
+ });
1348
+ }
1349
+ return {
1350
+ text: `Generated 5s motion transfer (${motionModel}) into project ${input.projectId} ` +
1351
+ `for $${(totalCents / 100).toFixed(2)} (${totalCents}¢).` +
1352
+ (input.prompt ? ` Prompt: "${input.prompt.slice(0, 60)}${input.prompt.length > 60 ? '...' : ''}"` : ''),
1353
+ data: {
1354
+ variant: costKey,
1355
+ motionModel,
1356
+ projectId: input.projectId,
1357
+ sourceVideoAssetId: input.sourceVideoAssetId,
1358
+ targetImageAssetId: input.targetImageAssetId,
1359
+ cost_cents: totalCents,
1360
+ cost_dollars: (totalCents / 100).toFixed(2),
1361
+ asset: result.asset,
1362
+ generationId: result.generationId,
1363
+ },
1364
+ };
1365
+ },
1366
+ };
1367
+ // ── Generation status (background mode) ─────────────────────────
1368
+ export const getGenerationStatus = {
1369
+ id: 'slates_get_generation_status',
1370
+ description: 'Poll one generation by id. Returns status (pending/processing/completed/failed/cancelled), the error message on failure, and the finished asset record (with id, code, filePath) on completion. Use after submitting any generate_* op with background=true; poll every 5-15s — video generations commonly take 1-5 minutes. Generations survive app restarts (the desktop resumes in-flight provider jobs on boot).',
1371
+ input: z.object({ generationId: z.string().uuid() }),
1372
+ async run(input, ctx) {
1373
+ const desktop = ctx.desktop();
1374
+ await desktop.requireCapability('background-generation', 'background generation');
1375
+ return ok(await desktop.get('/agent/generation/status', { id: input.generationId }));
1376
+ },
1377
+ };
1378
+ export const listGenerations = {
1379
+ id: 'slates_list_generations',
1380
+ description: "List recent and in-flight generations (newest first), optionally filtered by project and/or status. Use status='processing' to see everything still running, or no filter to review the recent history with costs and errors.",
1381
+ input: z.object({
1382
+ projectId: z.string().uuid().optional(),
1383
+ status: z.enum(['pending', 'processing', 'completed', 'failed', 'cancelled']).optional(),
1384
+ limit: z.number().int().min(1).max(100).optional(),
1385
+ }),
1386
+ async run(input, ctx) {
1387
+ const desktop = ctx.desktop();
1388
+ await desktop.requireCapability('background-generation', 'background generation');
1389
+ return ok(await desktop.get('/agent/generation/list', {
1390
+ projectId: input.projectId,
1391
+ status: input.status,
1392
+ limit: input.limit,
1393
+ }));
1394
+ },
1395
+ };
1396
+ export const getTimeline = {
1397
+ id: 'slates_get_timeline',
1398
+ description: 'Get (or lazily create) the single editing timeline for a Slates project, with all tracks, clips, markers, and a flat clipIndex mapping every clip back to its source asset (assetId + code + label). Frames are the unit of time; durationSec is provided. Call this before adding, reordering, or removing clips, and before exporting — it tells you the timeline id, frame rate, resolution, and current end frame.',
1399
+ input: z.object({ projectId: z.string().uuid() }),
1400
+ async run(input, ctx) {
1401
+ const desktop = ctx.desktop();
1402
+ await desktop.requireCapability('timeline', 'timeline editing');
1403
+ const r = await desktop.get('/agent/timeline', {
1404
+ projectId: input.projectId,
1405
+ });
1406
+ const t = r.timeline ?? {};
1407
+ const clipCount = r.clipIndex?.length ??
1408
+ (t.tracks ?? []).reduce((n, tr) => n + (tr.clips?.length ?? 0), 0);
1409
+ const durationSec = t.durationSec ?? r.durationSec;
1410
+ const dims = t.width && t.height ? `${t.width}x${t.height}` : '?';
1411
+ return ok(r, `Timeline for project ${input.projectId}: ${clipCount} clip(s) on ${t.tracks?.length ?? '?'} track(s), ` +
1412
+ `${durationSec ?? '?'}s at ${t.frameRate ?? '?'} fps, ${dims}.\n\n` +
1413
+ JSON.stringify(r, null, 2));
1414
+ },
1415
+ };
1416
+ export const addClipToTimeline = {
1417
+ id: 'slates_add_clip_to_timeline',
1418
+ description: "Append a video asset from the project to the project's timeline (or place it at an explicit startFrame). Defaults match the desktop UI: clip goes to the end of the first video track, full source duration, and an empty timeline auto-adopts the clip's resolution and frame rate. Optionally trim with sourceInFrame/sourceOutFrame (frames at the SOURCE fps). Use slates_get_timeline first to see current clips and pick positions. Only video assets are accepted.",
1419
+ input: z.object({
1420
+ projectId: z.string().uuid(),
1421
+ assetId: z.string().uuid().describe('Video asset already in the project.'),
1422
+ trackId: z.string().uuid().optional().describe('Target track. Default: the first video track.'),
1423
+ startFrame: z.number().int().min(0).optional().describe('Timeline frame to place the clip at. Default: append after the last clip.'),
1424
+ sourceInFrame: z.number().int().min(0).optional(),
1425
+ sourceOutFrame: z.number().int().min(1).optional(),
1426
+ }),
1427
+ async run(input, ctx) {
1428
+ const desktop = ctx.desktop();
1429
+ await desktop.requireCapability('timeline', 'timeline editing');
1430
+ const r = await desktop.post('/agent/timeline/add-clip', {
1431
+ projectId: input.projectId,
1432
+ assetId: input.assetId,
1433
+ trackId: input.trackId,
1434
+ startFrame: input.startFrame,
1435
+ sourceInFrame: input.sourceInFrame,
1436
+ sourceOutFrame: input.sourceOutFrame,
1437
+ });
1438
+ const clip = r.clip ?? {};
1439
+ // Code + label for the chat reference: prefer fields on the clip, fall
1440
+ // back to the clipIndex entry for this clip / its source asset.
1441
+ const idx = (r.clipIndex ?? []).find((c) => (clip.id && c.clipId === clip.id) || (!clip.id && clip.assetId && c.assetId === clip.assetId));
1442
+ const code = clip.code ?? idx?.code ?? null;
1443
+ const label = clip.label ?? idx?.label ?? null;
1444
+ const ref = code ? (label ? `${code} — ${label}` : code) : input.assetId;
1445
+ const frames = clip.durationFrames ??
1446
+ (clip.endFrame != null && clip.startFrame != null ? clip.endFrame - clip.startFrame : '?');
1447
+ return ok(r, `Added ${ref} to timeline at frame ${clip.startFrame ?? '?'} (${frames} frames long).`);
1448
+ },
1449
+ };
1450
+ export const reorderClips = {
1451
+ id: 'slates_reorder_clips',
1452
+ description: "Reorder the clips on one timeline track. Pass the COMPLETE list of the track's clip ids in the desired playback order; clips are repacked back-to-back from frame 0 (existing gaps are removed). Get current clip ids from slates_get_timeline.",
1453
+ input: z.object({
1454
+ trackId: z.string().uuid(),
1455
+ clipIds: z.array(z.string().uuid()).min(1),
1456
+ }),
1457
+ async run(input, ctx) {
1458
+ const desktop = ctx.desktop();
1459
+ await desktop.requireCapability('timeline', 'timeline editing');
1460
+ return ok(await desktop.post('/agent/timeline/reorder-clips', input));
1461
+ },
1462
+ };
1463
+ export const removeClip = {
1464
+ id: 'slates_remove_clip',
1465
+ description: "Remove a clip from the timeline (the source asset is untouched). Leaves a gap at the clip's old position — MP4 export renders gaps as black; call slates_reorder_clips afterwards to close gaps.",
1466
+ input: z.object({ clipId: z.string().uuid() }),
1467
+ async run(input, ctx) {
1468
+ const desktop = ctx.desktop();
1469
+ await desktop.requireCapability('timeline', 'timeline editing');
1470
+ return ok(await desktop.post('/agent/timeline/remove-clip', input));
1471
+ },
1472
+ };
1473
+ // ── Export ──────────────────────────────────────────────────────
1474
+ const ABSOLUTE_PATH_RE = /^([A-Za-z]:[\\/]|\/)/;
1475
+ export const exportVideo = {
1476
+ id: 'slates_export_video',
1477
+ description: "Render the project's timeline to an MP4 file on disk via the desktop app's ffmpeg pipeline (H.264 + AAC, gaps rendered as black). No dialogs — pass an absolute outputPath ending in .mp4. Fails if the file exists unless overwrite=true. Blocks until encoding finishes (can take minutes for long timelines) and returns the real file size and probed duration. After success, consider slates_reveal_file to show the file to the user. Requires at least one video clip on the timeline.",
1478
+ input: z
1479
+ .object({
1480
+ projectId: z.string().uuid().optional(),
1481
+ timelineId: z.string().uuid().optional(),
1482
+ outputPath: z
1483
+ .string()
1484
+ .min(5)
1485
+ .refine((p) => ABSOLUTE_PATH_RE.test(p), { message: 'outputPath must be absolute' })
1486
+ .describe('Absolute path for the rendered file, ending in .mp4 (e.g. C:\\Users\\you\\Videos\\ad.mp4 or /Users/you/ad.mp4). slates_get_project_directory gives a sensible default folder.'),
1487
+ overwrite: z.boolean().optional(),
1488
+ })
1489
+ .refine((d) => !!d.projectId !== !!d.timelineId, {
1490
+ message: 'Pass exactly one of projectId or timelineId',
1491
+ }),
1492
+ async run(input, ctx) {
1493
+ const desktop = ctx.desktop();
1494
+ await desktop.requireCapability('export', 'video export');
1495
+ const r = await desktop.post('/agent/timeline/export-video', {
1496
+ projectId: input.projectId,
1497
+ timelineId: input.timelineId,
1498
+ outputPath: input.outputPath,
1499
+ overwrite: input.overwrite,
1500
+ });
1501
+ const mb = r.fileSizeBytes != null ? (r.fileSizeBytes / (1024 * 1024)).toFixed(1) : '?';
1502
+ return ok(r, `Exported ${r.durationSec ?? '?'}s MP4 (${mb} MB) to ${r.outputPath ?? input.outputPath}.`);
1503
+ },
1504
+ };
1505
+ export const exportTimelineXml = {
1506
+ id: 'slates_export_timeline_xml',
1507
+ description: "Export the project's timeline as FCP7/XMEML XML — the file DaVinci Resolve imports directly (File → Import → Timeline) to recreate the edit with references to the original clip media on disk. This is the 'open in DaVinci' handoff path. Pass an absolute outputPath ending in .xml; fails if the file exists unless overwrite=true.",
1508
+ input: z
1509
+ .object({
1510
+ projectId: z.string().uuid().optional(),
1511
+ timelineId: z.string().uuid().optional(),
1512
+ outputPath: z
1513
+ .string()
1514
+ .min(5)
1515
+ .refine((p) => ABSOLUTE_PATH_RE.test(p), { message: 'outputPath must be absolute' })
1516
+ .describe('Absolute path for the XML file, ending in .xml (e.g. C:\\Users\\you\\Videos\\ad.xml or /Users/you/ad.xml).'),
1517
+ overwrite: z.boolean().optional(),
1518
+ })
1519
+ .refine((d) => !!d.projectId !== !!d.timelineId, {
1520
+ message: 'Pass exactly one of projectId or timelineId',
1521
+ }),
1522
+ async run(input, ctx) {
1523
+ const desktop = ctx.desktop();
1524
+ await desktop.requireCapability('export', 'timeline XML export');
1525
+ const r = await desktop.post('/agent/timeline/export-xml', {
1526
+ projectId: input.projectId,
1527
+ timelineId: input.timelineId,
1528
+ outputPath: input.outputPath,
1529
+ overwrite: input.overwrite,
1530
+ });
1531
+ return ok(r, `Exported timeline XML to ${r.outputPath ?? input.outputPath}. Import in DaVinci Resolve via File → Import → Timeline.`);
1532
+ },
1533
+ };
1534
+ export const revealFile = {
1535
+ id: 'slates_reveal_file',
1536
+ description: 'Open the OS file manager (Explorer/Finder) with the given file selected — use after slates_export_video / slates_export_timeline_xml so the user can see the file you wrote. Absolute path required; the file must exist.',
1537
+ input: z.object({ path: z.string().min(3) }),
1538
+ async run(input, ctx) {
1539
+ const desktop = ctx.desktop();
1540
+ await desktop.requireCapability('export', 'revealing files in the file manager');
1541
+ return ok(await desktop.post('/agent/reveal-file', input));
1542
+ },
1543
+ };
1544
+ // ── CRUD completion (agent API v1 routes — no capability check) ─
1545
+ export const updateProject = {
1546
+ id: 'slates_update_project',
1547
+ description: 'Update a Slates project\'s name and/or description.',
1548
+ input: z.object({
1549
+ id: z.string().uuid(),
1550
+ name: z.string().min(1).max(120).optional(),
1551
+ description: z.string().optional(),
1552
+ }),
1553
+ async run(input, ctx) {
1554
+ return ok(await ctx.desktop().post('/agent/projects/update', {
1555
+ id: input.id,
1556
+ data: { name: input.name, description: input.description },
1557
+ }));
1558
+ },
1559
+ };
1560
+ export const deleteProject = {
1561
+ id: 'slates_delete_project',
1562
+ description: 'Delete a Slates project. DESTRUCTIVE — permanently removes the project with all its assets, storyboards, and media files. Requires confirm=true after explicit user OK.',
1563
+ input: z.object({
1564
+ id: z.string().uuid(),
1565
+ confirm: z.boolean().optional().describe('Set true only after the user explicitly OKs the deletion.'),
1566
+ }),
1567
+ async run(input, ctx) {
1568
+ if (!input.confirm) {
1569
+ return ok({
1570
+ requires_confirm: true,
1571
+ message: 'Deleting a project permanently removes all its assets, storyboards, and media files. Re-call with confirm=true after explicit user OK.',
1572
+ });
1573
+ }
1574
+ return ok(await ctx.desktop().post('/agent/projects/delete', { id: input.id }));
1575
+ },
1576
+ };
1577
+ export const getProjectDirectory = {
1578
+ id: 'slates_get_project_directory',
1579
+ description: 'Get the absolute on-disk folder of a Slates project — useful for choosing an export outputPath default (e.g. <dir>/exports/final.mp4).',
1580
+ input: z.object({ id: z.string().uuid() }),
1581
+ async run(input, ctx) {
1582
+ return ok(await ctx.desktop().get('/agent/projects/location', { id: input.id }));
1583
+ },
1584
+ };
1585
+ export const deleteAsset = {
1586
+ id: 'slates_delete_asset',
1587
+ description: 'Delete an asset from its project. Permanent — also deletes the media file from disk.',
1588
+ input: z.object({ id: z.string().uuid() }),
1589
+ async run(input, ctx) {
1590
+ return ok(await ctx.desktop().post('/agent/assets/delete', { id: input.id }));
1591
+ },
1592
+ };
1593
+ export const renameFolder = {
1594
+ id: 'slates_rename_folder',
1595
+ description: 'Rename an asset folder.',
1596
+ input: z.object({
1597
+ folderId: z.string().uuid(),
1598
+ name: z.string().min(1).max(120),
1599
+ }),
1600
+ async run(input, ctx) {
1601
+ return ok(await ctx.desktop().post('/agent/folders/rename', { id: input.folderId, name: input.name }));
1602
+ },
1603
+ };
1604
+ export const deleteFolder = {
1605
+ id: 'slates_delete_folder',
1606
+ description: 'Delete an asset folder.',
1607
+ input: z.object({ folderId: z.string().uuid() }),
1608
+ async run(input, ctx) {
1609
+ return ok(await ctx.desktop().post('/agent/folders/delete', { id: input.folderId }));
1610
+ },
1611
+ };
1612
+ export const setFolderCover = {
1613
+ id: 'slates_set_folder_cover',
1614
+ description: 'Set the cover image of an asset folder (or clear it with assetId=null).',
1615
+ input: z.object({
1616
+ folderId: z.string().uuid(),
1617
+ assetId: z.string().uuid().nullable(),
1618
+ }),
1619
+ async run(input, ctx) {
1620
+ return ok(await ctx.desktop().post('/agent/folders/set-cover', input));
1621
+ },
1622
+ };
1623
+ export const updateCharacter = {
1624
+ id: 'slates_update_character',
1625
+ description: 'Update a character\'s name, description, or style. (Turnaround / expression image slots have their own dedicated set_character_* ops.)',
1626
+ input: z.object({
1627
+ characterId: z.string().uuid(),
1628
+ name: z.string().min(1).max(120).optional(),
1629
+ description: z.string().optional(),
1630
+ style: z.enum(['realistic', 'anime', 'pixar', 'comic-book']).optional(),
1631
+ }),
1632
+ async run(input, ctx) {
1633
+ return ok(await ctx.desktop().post('/agent/characters/update', {
1634
+ id: input.characterId,
1635
+ data: { name: input.name, description: input.description, style: input.style },
1636
+ }));
1637
+ },
1638
+ };
1639
+ export const deleteCharacter = {
1640
+ id: 'slates_delete_character',
1641
+ description: 'Delete a character from its project.',
1642
+ input: z.object({ characterId: z.string().uuid() }),
1643
+ async run(input, ctx) {
1644
+ return ok(await ctx.desktop().post('/agent/characters/delete', { id: input.characterId }));
1645
+ },
1646
+ };
1647
+ export const updateEnvironment = {
1648
+ id: 'slates_update_environment',
1649
+ description: 'Update an environment\'s name, description, style, or bound grid asset (gridAssetId=null clears it).',
1650
+ input: z.object({
1651
+ environmentId: z.string().uuid(),
1652
+ name: z.string().min(1).max(120).optional(),
1653
+ description: z.string().optional(),
1654
+ style: z.enum(['realistic', 'anime', 'pixar', 'comic-book']).optional(),
1655
+ gridAssetId: z.string().uuid().nullable().optional(),
1656
+ }),
1657
+ async run(input, ctx) {
1658
+ return ok(await ctx.desktop().post('/agent/environments/update', {
1659
+ id: input.environmentId,
1660
+ data: {
1661
+ name: input.name,
1662
+ description: input.description,
1663
+ style: input.style,
1664
+ gridAssetId: input.gridAssetId,
1665
+ },
1666
+ }));
1667
+ },
1668
+ };
1669
+ export const deleteEnvironment = {
1670
+ id: 'slates_delete_environment',
1671
+ description: 'Delete an environment from its project.',
1672
+ input: z.object({ environmentId: z.string().uuid() }),
1673
+ async run(input, ctx) {
1674
+ return ok(await ctx.desktop().post('/agent/environments/delete', { id: input.environmentId }));
1675
+ },
1676
+ };
1677
+ export const listStyles = {
1678
+ id: 'slates_list_styles',
1679
+ description: 'List visual styles in a Slates project.',
1680
+ input: z.object({ projectId: z.string().uuid() }),
1681
+ async run(input, ctx) {
1682
+ return ok(await ctx.desktop().get('/agent/styles', { projectId: input.projectId }));
1683
+ },
1684
+ };
1685
+ export const createStyle = {
1686
+ id: 'slates_create_style',
1687
+ description: 'Create a new visual style in a Slates project.',
1688
+ input: z.object({
1689
+ projectId: z.string().uuid(),
1690
+ name: z.string().min(1).max(120),
1691
+ description: z.string().optional(),
1692
+ }),
1693
+ async run(input, ctx) {
1694
+ return ok(await ctx.desktop().post('/agent/styles', input));
1695
+ },
1696
+ };
1697
+ export const updateStyle = {
1698
+ id: 'slates_update_style',
1699
+ description: 'Update a style\'s name, description, or bound reference image (imageAssetId=null clears it).',
1700
+ input: z.object({
1701
+ styleId: z.string().uuid(),
1702
+ name: z.string().min(1).max(120).optional(),
1703
+ description: z.string().optional(),
1704
+ imageAssetId: z.string().uuid().nullable().optional(),
1705
+ }),
1706
+ async run(input, ctx) {
1707
+ return ok(await ctx.desktop().post('/agent/styles/update', {
1708
+ id: input.styleId,
1709
+ data: { name: input.name, description: input.description, imageAssetId: input.imageAssetId },
1710
+ }));
1711
+ },
1712
+ };
1713
+ export const deleteStyle = {
1714
+ id: 'slates_delete_style',
1715
+ description: 'Delete a visual style from its project.',
1716
+ input: z.object({ styleId: z.string().uuid() }),
1717
+ async run(input, ctx) {
1718
+ return ok(await ctx.desktop().post('/agent/styles/delete', { id: input.styleId }));
1719
+ },
1720
+ };
1721
+ export const updateStoryboard = {
1722
+ id: 'slates_update_storyboard',
1723
+ description: 'Update a storyboard\'s name and/or description.',
1724
+ input: z.object({
1725
+ storyboardId: z.string().uuid(),
1726
+ name: z.string().min(1).max(120).optional(),
1727
+ description: z.string().optional(),
1728
+ }),
1729
+ async run(input, ctx) {
1730
+ return ok(await ctx.desktop().post('/agent/storyboards/update', {
1731
+ id: input.storyboardId,
1732
+ data: { name: input.name, description: input.description },
1733
+ }));
1734
+ },
1735
+ };
1736
+ export const deleteStoryboard = {
1737
+ id: 'slates_delete_storyboard',
1738
+ description: 'Delete a storyboard with all its scenes and frames (the referenced assets are untouched).',
1739
+ input: z.object({ storyboardId: z.string().uuid() }),
1740
+ async run(input, ctx) {
1741
+ return ok(await ctx.desktop().post('/agent/storyboards/delete', { id: input.storyboardId }));
1742
+ },
1743
+ };
1744
+ export const updateScene = {
1745
+ id: 'slates_update_scene',
1746
+ description: 'Update a scene\'s name and/or position within its storyboard.',
1747
+ input: z.object({
1748
+ sceneId: z.string().uuid(),
1749
+ name: z.string().min(1).max(120).optional(),
1750
+ position: z.number().int().min(0).optional(),
1751
+ }),
1752
+ async run(input, ctx) {
1753
+ return ok(await ctx.desktop().post('/agent/scenes/update', {
1754
+ id: input.sceneId,
1755
+ data: { name: input.name, position: input.position },
1756
+ }));
1757
+ },
1758
+ };
1759
+ export const deleteScene = {
1760
+ id: 'slates_delete_scene',
1761
+ description: 'Delete a scene (and its frames) from a storyboard.',
1762
+ input: z.object({ sceneId: z.string().uuid() }),
1763
+ async run(input, ctx) {
1764
+ return ok(await ctx.desktop().post('/agent/scenes/delete', { id: input.sceneId }));
1765
+ },
1766
+ };
1767
+ export const reorderScenes = {
1768
+ id: 'slates_reorder_scenes',
1769
+ description: 'Reorder the scenes of a storyboard. Pass the COMPLETE list of the storyboard\'s scene ids in the desired order.',
1770
+ input: z.object({
1771
+ storyboardId: z.string().uuid(),
1772
+ sceneIds: z.array(z.string().uuid()).min(1),
1773
+ }),
1774
+ async run(input, ctx) {
1775
+ return ok(await ctx.desktop().post('/agent/scenes/reorder', input));
1776
+ },
1777
+ };
1778
+ export const updateFrame = {
1779
+ id: 'slates_update_frame',
1780
+ description: 'Update a frame: shot label, notes, bound asset (assetId=null unbinds), scene, position, frameType (first/last/ingredient, null clears), or motion prompt (null clears).',
1781
+ input: z.object({
1782
+ frameId: z.string().uuid(),
1783
+ shotLabel: z.string().optional(),
1784
+ notes: z.string().optional(),
1785
+ assetId: z.string().uuid().nullable().optional(),
1786
+ sceneId: z.string().uuid().nullable().optional(),
1787
+ position: z.number().int().min(0).optional(),
1788
+ frameType: z.enum(['first', 'last', 'ingredient']).nullable().optional(),
1789
+ motionPrompt: z.string().nullable().optional(),
1790
+ }),
1791
+ async run(input, ctx) {
1792
+ return ok(await ctx.desktop().post('/agent/frames/update', {
1793
+ id: input.frameId,
1794
+ data: {
1795
+ shotLabel: input.shotLabel,
1796
+ notes: input.notes,
1797
+ assetId: input.assetId,
1798
+ sceneId: input.sceneId,
1799
+ position: input.position,
1800
+ frameType: input.frameType,
1801
+ motionPrompt: input.motionPrompt,
1802
+ },
1803
+ }));
1804
+ },
1805
+ };
1806
+ export const deleteFrame = {
1807
+ id: 'slates_delete_frame',
1808
+ description: 'Delete a frame from its scene (the referenced asset is untouched).',
1809
+ input: z.object({ frameId: z.string().uuid() }),
1810
+ async run(input, ctx) {
1811
+ return ok(await ctx.desktop().post('/agent/frames/delete', { id: input.frameId }));
1812
+ },
1813
+ };
1814
+ // ── Prompting guides (local lookup — no transport) ──────────────
1815
+ // Model-id → guide-name aliasing. Order matters: kling-mc-* (motion
1816
+ // transfer) must match before the generic kling-v3* check.
1817
+ function resolveGuideTopic(topic) {
1818
+ const t = topic.trim().toLowerCase();
1819
+ if (SKILLS[t])
1820
+ return t;
1821
+ if (t.startsWith('nano-banana'))
1822
+ return 'slates-prompting-nano-banana-2';
1823
+ if (t.startsWith('flux'))
1824
+ return 'slates-prompting-flux-2-max';
1825
+ if (t.startsWith('seedream'))
1826
+ return 'slates-prompting-seedream-5-lite';
1827
+ if (t.startsWith('veo'))
1828
+ return 'slates-prompting-veo-3';
1829
+ if (t.startsWith('kling-mc'))
1830
+ return 'slates-prompting-motion-transfer';
1831
+ if (t.startsWith('kling-v3'))
1832
+ return 'slates-prompting-kling-v3';
1833
+ if (t.startsWith('seedance'))
1834
+ return 'slates-prompting-seedance';
1835
+ if (t.startsWith('avatar-') || t.includes('lip-sync'))
1836
+ return 'slates-prompting-lip-sync';
1837
+ return null;
1838
+ }
1839
+ export const getPromptingGuide = {
1840
+ id: 'slates_get_prompting_guide',
1841
+ description: "Return the full markdown of a bundled Slates prompting/workflow guide. MCP-only clients (Claude Desktop, Smithery) don't get the CLI-installed skill files — call this instead. Accepts a guide name or a model id (e.g. 'veo-3.1-fast', 'kling-v3.0-pro', 'seedance-2-std', 'nano-banana-2') which maps to the right guide. ALWAYS read 'slates-cost-discipline' plus the relevant model guide before your first generation in a session.",
1842
+ input: z.object({
1843
+ topic: z
1844
+ .string()
1845
+ .min(1)
1846
+ .describe('Guide name or model id. Guides: slates-cost-discipline, slates-prompting-nano-banana-2, slates-prompting-veo-3, slates-prompting-kling-v3, slates-prompting-seedance, slates-prompting-lip-sync, slates-prompting-motion-transfer, slates-prompting-flux-2-max, slates-prompting-seedream-5-lite, slates-edit-and-iterate, slates-vision-feedback-loop, slates-character-turnaround, slates-storyboard-from-script, slates-direct-response-ad, slates-one-prompt-film'),
1847
+ }),
1848
+ async run(input) {
1849
+ const resolved = resolveGuideTopic(input.topic);
1850
+ const content = resolved ? SKILLS[resolved] : undefined;
1851
+ if (!resolved || content === undefined) {
1852
+ throw new Error(`Unknown guide topic: ${input.topic}. Valid topics: ${Object.keys(SKILLS).sort().join(', ')}`);
1853
+ }
1854
+ return {
1855
+ text: content,
1856
+ data: { topic: resolved, bytes: Buffer.byteLength(content, 'utf8') },
414
1857
  };
415
1858
  },
416
1859
  };
@@ -426,6 +1869,8 @@ export const ALL_OPERATIONS = [
426
1869
  getProject,
427
1870
  listAssets,
428
1871
  getAssetImage,
1872
+ getAssetsBatch,
1873
+ getAssetVideoFrames,
429
1874
  uploadReferenceImage,
430
1875
  listFolders,
431
1876
  createFolder,
@@ -442,5 +1887,41 @@ export const ALL_OPERATIONS = [
442
1887
  addScene,
443
1888
  addFrame,
444
1889
  generateImage,
1890
+ generateVideo,
1891
+ generateLipSync,
1892
+ generateMotionTransfer,
1893
+ editImage,
1894
+ getGenerationStatus,
1895
+ listGenerations,
1896
+ getTimeline,
1897
+ addClipToTimeline,
1898
+ reorderClips,
1899
+ removeClip,
1900
+ exportVideo,
1901
+ exportTimelineXml,
1902
+ revealFile,
1903
+ updateProject,
1904
+ deleteProject,
1905
+ getProjectDirectory,
1906
+ deleteAsset,
1907
+ renameFolder,
1908
+ deleteFolder,
1909
+ setFolderCover,
1910
+ updateCharacter,
1911
+ deleteCharacter,
1912
+ updateEnvironment,
1913
+ deleteEnvironment,
1914
+ listStyles,
1915
+ createStyle,
1916
+ updateStyle,
1917
+ deleteStyle,
1918
+ updateStoryboard,
1919
+ deleteStoryboard,
1920
+ updateScene,
1921
+ deleteScene,
1922
+ reorderScenes,
1923
+ updateFrame,
1924
+ deleteFrame,
1925
+ getPromptingGuide,
445
1926
  ];
446
1927
  //# sourceMappingURL=index.js.map