@vibeframe/cli 0.27.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/LICENSE +21 -0
  2. package/dist/agent/adapters/index.d.ts +1 -0
  3. package/dist/agent/adapters/index.d.ts.map +1 -1
  4. package/dist/agent/adapters/index.js +5 -0
  5. package/dist/agent/adapters/index.js.map +1 -1
  6. package/dist/agent/adapters/openrouter.d.ts +16 -0
  7. package/dist/agent/adapters/openrouter.d.ts.map +1 -0
  8. package/dist/agent/adapters/openrouter.js +100 -0
  9. package/dist/agent/adapters/openrouter.js.map +1 -0
  10. package/dist/agent/types.d.ts +1 -1
  11. package/dist/agent/types.d.ts.map +1 -1
  12. package/dist/commands/agent.d.ts.map +1 -1
  13. package/dist/commands/agent.js +3 -1
  14. package/dist/commands/agent.js.map +1 -1
  15. package/dist/commands/setup.js +5 -2
  16. package/dist/commands/setup.js.map +1 -1
  17. package/dist/config/schema.d.ts +2 -1
  18. package/dist/config/schema.d.ts.map +1 -1
  19. package/dist/config/schema.js +2 -0
  20. package/dist/config/schema.js.map +1 -1
  21. package/dist/index.js +0 -0
  22. package/package.json +16 -12
  23. package/.turbo/turbo-build.log +0 -4
  24. package/.turbo/turbo-lint.log +0 -21
  25. package/.turbo/turbo-test.log +0 -689
  26. package/src/agent/adapters/claude.ts +0 -143
  27. package/src/agent/adapters/gemini.ts +0 -159
  28. package/src/agent/adapters/index.ts +0 -61
  29. package/src/agent/adapters/ollama.ts +0 -231
  30. package/src/agent/adapters/openai.ts +0 -116
  31. package/src/agent/adapters/xai.ts +0 -119
  32. package/src/agent/index.ts +0 -251
  33. package/src/agent/memory/index.ts +0 -151
  34. package/src/agent/prompts/system.ts +0 -106
  35. package/src/agent/tools/ai-editing.ts +0 -845
  36. package/src/agent/tools/ai-generation.ts +0 -1073
  37. package/src/agent/tools/ai-pipeline.ts +0 -1055
  38. package/src/agent/tools/ai.ts +0 -21
  39. package/src/agent/tools/batch.ts +0 -429
  40. package/src/agent/tools/e2e.test.ts +0 -545
  41. package/src/agent/tools/export.ts +0 -184
  42. package/src/agent/tools/filesystem.ts +0 -237
  43. package/src/agent/tools/index.ts +0 -150
  44. package/src/agent/tools/integration.test.ts +0 -775
  45. package/src/agent/tools/media.ts +0 -697
  46. package/src/agent/tools/project.ts +0 -313
  47. package/src/agent/tools/timeline.ts +0 -951
  48. package/src/agent/types.ts +0 -68
  49. package/src/commands/agent.ts +0 -340
  50. package/src/commands/ai-analyze.ts +0 -429
  51. package/src/commands/ai-animated-caption.ts +0 -390
  52. package/src/commands/ai-audio.ts +0 -941
  53. package/src/commands/ai-broll.ts +0 -490
  54. package/src/commands/ai-edit-cli.ts +0 -658
  55. package/src/commands/ai-edit.ts +0 -1542
  56. package/src/commands/ai-fill-gaps.ts +0 -566
  57. package/src/commands/ai-helpers.ts +0 -65
  58. package/src/commands/ai-highlights.ts +0 -1303
  59. package/src/commands/ai-image.ts +0 -761
  60. package/src/commands/ai-motion.ts +0 -347
  61. package/src/commands/ai-narrate.ts +0 -451
  62. package/src/commands/ai-review.ts +0 -309
  63. package/src/commands/ai-script-pipeline-cli.ts +0 -1710
  64. package/src/commands/ai-script-pipeline.ts +0 -1365
  65. package/src/commands/ai-suggest-edit.ts +0 -264
  66. package/src/commands/ai-video-fx.ts +0 -445
  67. package/src/commands/ai-video.ts +0 -915
  68. package/src/commands/ai-viral.ts +0 -595
  69. package/src/commands/ai-visual-fx.ts +0 -601
  70. package/src/commands/ai.test.ts +0 -627
  71. package/src/commands/ai.ts +0 -307
  72. package/src/commands/analyze.ts +0 -282
  73. package/src/commands/audio.ts +0 -644
  74. package/src/commands/batch.test.ts +0 -279
  75. package/src/commands/batch.ts +0 -440
  76. package/src/commands/detect.ts +0 -329
  77. package/src/commands/doctor.ts +0 -237
  78. package/src/commands/edit-cmd.ts +0 -1014
  79. package/src/commands/export.ts +0 -918
  80. package/src/commands/generate.ts +0 -2146
  81. package/src/commands/media.ts +0 -177
  82. package/src/commands/output.ts +0 -142
  83. package/src/commands/pipeline.ts +0 -398
  84. package/src/commands/project.test.ts +0 -127
  85. package/src/commands/project.ts +0 -149
  86. package/src/commands/sanitize.ts +0 -60
  87. package/src/commands/schema.ts +0 -130
  88. package/src/commands/setup.ts +0 -509
  89. package/src/commands/timeline.test.ts +0 -499
  90. package/src/commands/timeline.ts +0 -529
  91. package/src/commands/validate.ts +0 -77
  92. package/src/config/config.test.ts +0 -197
  93. package/src/config/index.ts +0 -125
  94. package/src/config/schema.ts +0 -82
  95. package/src/engine/index.ts +0 -2
  96. package/src/engine/project.test.ts +0 -702
  97. package/src/engine/project.ts +0 -439
  98. package/src/index.ts +0 -146
  99. package/src/utils/api-key.test.ts +0 -41
  100. package/src/utils/api-key.ts +0 -247
  101. package/src/utils/audio.ts +0 -83
  102. package/src/utils/exec-safe.ts +0 -75
  103. package/src/utils/first-run.ts +0 -52
  104. package/src/utils/provider-resolver.ts +0 -56
  105. package/src/utils/remotion.ts +0 -951
  106. package/src/utils/subtitle.test.ts +0 -227
  107. package/src/utils/subtitle.ts +0 -169
  108. package/src/utils/tty.ts +0 -196
  109. package/tsconfig.json +0 -20
@@ -1,1073 +0,0 @@
1
- /**
2
- * @module ai-generation
3
- * @description Agent tools for AI asset generation (image, video, TTS, SFX, music,
4
- * storyboard, motion). Wraps providers for agent use. Some features require
5
- * async polling -- tool returns immediately with task status.
6
- *
7
- * ## Tools: generate_image, generate_video, generate_speech, generate_sound_effect, generate_music, generate_storyboard, generate_motion
8
- * ## Dependencies: OpenAI, Gemini, Runway, Kling, ElevenLabs (TTS/SFX/Music), Replicate, Claude, Remotion
9
- * @see MODELS.md for the Single Source of Truth (SSOT) on supported providers/models
10
- */
11
-
12
- import { writeFile, readFile } from "node:fs/promises";
13
- import { resolve } from "node:path";
14
- import type { ToolRegistry, ToolHandler } from "./index.js";
15
- import type { ToolDefinition, ToolResult } from "../types.js";
16
- import { getApiKeyFromConfig } from "../../config/index.js";
17
- import { downloadVideo } from "../../commands/ai-helpers.js";
18
- import { sanitizeAIResult } from "../../commands/sanitize.js";
19
-
20
- // Helper to get timestamp for filenames
21
- function getTimestamp(): string {
22
- return Date.now().toString();
23
- }
24
-
25
- // ============================================================================
26
- // Tool Definitions
27
- // ============================================================================
28
-
29
- const imageDef: ToolDefinition = {
30
- name: "generate_image",
31
- description: "Generate an image using AI (OpenAI GPT Image 1.5, Gemini, or xAI Grok)",
32
- parameters: {
33
- type: "object",
34
- properties: {
35
- prompt: {
36
- type: "string",
37
- description: "Image generation prompt",
38
- },
39
- output: {
40
- type: "string",
41
- description: "Output file path (default: generated-{timestamp}.png)",
42
- },
43
- provider: {
44
- type: "string",
45
- description: "Provider to use: openai (GPT Image 1.5), gemini (Nano Banana), grok (Grok Imagine). 'dalle' is deprecated, use 'openai' instead.",
46
- enum: ["openai", "dalle", "gemini", "grok"],
47
- },
48
- size: {
49
- type: "string",
50
- description: "Image size (1024x1024, 1536x1024, 1024x1536)",
51
- enum: ["1024x1024", "1536x1024", "1024x1536"],
52
- },
53
- },
54
- required: ["prompt"],
55
- },
56
- };
57
-
58
- const videoDef: ToolDefinition = {
59
- name: "generate_video",
60
- description: "Generate video using AI. Supports Grok Imagine (default, native audio), Runway Gen-4.5 (text/image-to-video) and gen4_turbo (image-to-video), Kling (text/image-to-video), and Veo (text/image-to-video) via provider selection.",
61
- parameters: {
62
- type: "object",
63
- properties: {
64
- prompt: {
65
- type: "string",
66
- description: "Video generation prompt describing the motion/animation",
67
- },
68
- provider: {
69
- type: "string",
70
- description: "Video provider to use (default: grok)",
71
- enum: ["grok", "runway", "kling", "veo"],
72
- },
73
- image: {
74
- type: "string",
75
- description: "Input image path. Required for runway gen4_turbo, optional for runway gen4.5/kling/veo.",
76
- },
77
- runwayModel: {
78
- type: "string",
79
- description: "Runway model: gen4.5 (default, text+image-to-video) or gen4_turbo (image-to-video only)",
80
- enum: ["gen4.5", "gen4_turbo"],
81
- },
82
- output: {
83
- type: "string",
84
- description: "Output file path",
85
- },
86
- duration: {
87
- type: "number",
88
- description: "Video duration in seconds",
89
- },
90
- mode: {
91
- type: "string",
92
- description: "Quality mode for Kling (std or pro)",
93
- enum: ["std", "pro"],
94
- },
95
- negativePrompt: {
96
- type: "string",
97
- description: "What to avoid in the generated video (Veo only)",
98
- },
99
- resolution: {
100
- type: "string",
101
- description: "Video resolution (Veo only)",
102
- enum: ["720p", "1080p", "4k"],
103
- },
104
- },
105
- required: ["prompt"],
106
- },
107
- };
108
-
109
- const ttsDef: ToolDefinition = {
110
- name: "generate_speech",
111
- description: "Generate speech from text using ElevenLabs",
112
- parameters: {
113
- type: "object",
114
- properties: {
115
- text: {
116
- type: "string",
117
- description: "Text to convert to speech",
118
- },
119
- output: {
120
- type: "string",
121
- description: "Output audio file path",
122
- },
123
- voice: {
124
- type: "string",
125
- description: "Voice ID or name",
126
- },
127
- },
128
- required: ["text"],
129
- },
130
- };
131
-
132
- const sfxDef: ToolDefinition = {
133
- name: "generate_sound_effect",
134
- description: "Generate sound effects using ElevenLabs",
135
- parameters: {
136
- type: "object",
137
- properties: {
138
- prompt: {
139
- type: "string",
140
- description: "Sound effect description",
141
- },
142
- output: {
143
- type: "string",
144
- description: "Output audio file path",
145
- },
146
- duration: {
147
- type: "number",
148
- description: "Duration in seconds",
149
- },
150
- },
151
- required: ["prompt"],
152
- },
153
- };
154
-
155
- const musicDef: ToolDefinition = {
156
- name: "generate_music",
157
- description: "Generate music using AI. Default: ElevenLabs (synchronous, up to 10min). Alternative: Replicate/MusicGen (async, max 30s).",
158
- parameters: {
159
- type: "object",
160
- properties: {
161
- prompt: {
162
- type: "string",
163
- description: "Music description/prompt",
164
- },
165
- output: {
166
- type: "string",
167
- description: "Output audio file path",
168
- },
169
- duration: {
170
- type: "number",
171
- description: "Duration in seconds (elevenlabs: 3-600, replicate: 1-30)",
172
- },
173
- provider: {
174
- type: "string",
175
- description: "Provider: elevenlabs (default, synchronous, up to 10min) or replicate (async, max 30s)",
176
- enum: ["elevenlabs", "replicate"],
177
- },
178
- instrumental: {
179
- type: "boolean",
180
- description: "Force instrumental music with no vocals (ElevenLabs only)",
181
- },
182
- },
183
- required: ["prompt"],
184
- },
185
- };
186
-
187
- const storyboardDef: ToolDefinition = {
188
- name: "generate_storyboard",
189
- description: "Generate a storyboard from a script using Claude",
190
- parameters: {
191
- type: "object",
192
- properties: {
193
- script: {
194
- type: "string",
195
- description: "Video script or concept",
196
- },
197
- targetDuration: {
198
- type: "number",
199
- description: "Target video duration in seconds",
200
- },
201
- output: {
202
- type: "string",
203
- description: "Output JSON file path",
204
- },
205
- creativity: {
206
- type: "string",
207
- description: "Creativity level: 'low' (default, consistent scenes) or 'high' (varied, unexpected scenes)",
208
- enum: ["low", "high"],
209
- },
210
- },
211
- required: ["script"],
212
- },
213
- };
214
-
215
- const motionDef: ToolDefinition = {
216
- name: "generate_motion",
217
- description:
218
- "Generate motion graphics using Claude + Remotion. Can render to video and composite onto existing footage. " +
219
- "Without --video: generates and renders a standalone motion graphic. With --video: renders and composites onto the base video.",
220
- parameters: {
221
- type: "object",
222
- properties: {
223
- description: {
224
- type: "string",
225
- description: "Natural language description of the motion graphic (e.g., 'cinematic title card with fade-in')",
226
- },
227
- video: {
228
- type: "string",
229
- description: "Base video path to composite the motion graphic onto (triggers render + composite)",
230
- },
231
- output: {
232
- type: "string",
233
- description: "Output file path (.mp4 if compositing, .webm if render-only, .tsx if code-only)",
234
- },
235
- duration: {
236
- type: "number",
237
- description: "Duration in seconds (default: 5)",
238
- },
239
- width: {
240
- type: "number",
241
- description: "Width in pixels (default: 1920)",
242
- },
243
- height: {
244
- type: "number",
245
- description: "Height in pixels (default: 1080)",
246
- },
247
- style: {
248
- type: "string",
249
- description: "Style preset",
250
- enum: ["minimal", "corporate", "playful", "cinematic"],
251
- },
252
- },
253
- required: ["description"],
254
- },
255
- };
256
-
257
- // ============================================================================
258
- // Tool Handlers
259
- // ============================================================================
260
-
261
- const generateImage: ToolHandler = async (args, context): Promise<ToolResult> => {
262
- const prompt = args.prompt as string;
263
- const provider = (args.provider as string) || "gemini";
264
- const output = (args.output as string) || `generated-${getTimestamp()}.png`;
265
- const size = (args.size as string) || "1024x1024";
266
-
267
- try {
268
- let providerKey: string;
269
-
270
- switch (provider) {
271
- case "gemini":
272
- providerKey = "google";
273
- break;
274
- case "openai":
275
- case "dalle": // backward compatibility
276
- providerKey = "openai";
277
- break;
278
- case "grok":
279
- providerKey = "xai";
280
- break;
281
- default:
282
- providerKey = "openai";
283
- }
284
-
285
- const apiKey = await getApiKeyFromConfig(providerKey);
286
- if (!apiKey) {
287
- return {
288
- toolCallId: "",
289
- success: false,
290
- output: "",
291
- error: `API key required for ${provider}. Configure via 'vibe setup'.`,
292
- };
293
- }
294
-
295
- const outputPath = resolve(context.workingDirectory, output);
296
-
297
- if (provider === "dalle" || provider === "openai") {
298
- const { OpenAIImageProvider } = await import("@vibeframe/ai-providers");
299
- const openaiImage = new OpenAIImageProvider();
300
- await openaiImage.initialize({ apiKey });
301
-
302
- const result = await openaiImage.generateImage(prompt, {
303
- size: size as "1024x1024" | "1536x1024" | "1024x1536",
304
- });
305
-
306
- if (!result.success || !result.images || result.images.length === 0) {
307
- return {
308
- toolCallId: "",
309
- success: false,
310
- output: "",
311
- error: `Image generation failed: ${result.error || "No image generated"}`,
312
- };
313
- }
314
-
315
- // Save image (handle both URL and base64)
316
- const image = result.images[0];
317
- let buffer: Buffer;
318
- if (image.url) {
319
- const response = await fetch(image.url);
320
- buffer = Buffer.from(await response.arrayBuffer());
321
- } else if (image.base64) {
322
- buffer = Buffer.from(image.base64, "base64");
323
- } else {
324
- return {
325
- toolCallId: "",
326
- success: false,
327
- output: "",
328
- error: "Image generated but no URL or base64 data returned",
329
- };
330
- }
331
- await writeFile(outputPath, buffer);
332
- } else if (provider === "gemini") {
333
- const { GeminiProvider } = await import("@vibeframe/ai-providers");
334
- const gemini = new GeminiProvider();
335
- await gemini.initialize({ apiKey });
336
-
337
- const result = await gemini.generateImage(prompt);
338
-
339
- if (!result.success || !result.images || result.images.length === 0) {
340
- return {
341
- toolCallId: "",
342
- success: false,
343
- output: "",
344
- error: `Image generation failed: ${result.error || "No image generated"}`,
345
- };
346
- }
347
-
348
- // Gemini returns base64
349
- const image = result.images[0];
350
- if (image.base64) {
351
- const buffer = Buffer.from(image.base64, "base64");
352
- await writeFile(outputPath, buffer);
353
- }
354
- } else if (provider === "grok") {
355
- const { GrokProvider } = await import("@vibeframe/ai-providers");
356
- const grok = new GrokProvider();
357
- await grok.initialize({ apiKey });
358
-
359
- const result = await grok.generateImage(prompt);
360
-
361
- if (!result.success || !result.images || result.images.length === 0) {
362
- return {
363
- toolCallId: "",
364
- success: false,
365
- output: "",
366
- error: `Image generation failed: ${result.error || "No image generated"}`,
367
- };
368
- }
369
-
370
- // Grok returns URLs
371
- const image = result.images[0];
372
- if (image.url) {
373
- const response = await fetch(image.url);
374
- const buffer = Buffer.from(await response.arrayBuffer());
375
- await writeFile(outputPath, buffer);
376
- } else if (image.base64) {
377
- const buffer = Buffer.from(image.base64, "base64");
378
- await writeFile(outputPath, buffer);
379
- }
380
- }
381
-
382
- return {
383
- toolCallId: "",
384
- success: true,
385
- output: `Image generated: ${output}\nPrompt: ${prompt}\nProvider: ${provider}\nSize: ${size}`,
386
- };
387
- } catch (error) {
388
- return {
389
- toolCallId: "",
390
- success: false,
391
- output: "",
392
- error: `Failed to generate image: ${error instanceof Error ? error.message : String(error)}`,
393
- };
394
- }
395
- };
396
-
397
- const generateVideo: ToolHandler = async (args, context): Promise<ToolResult> => {
398
- const prompt = args.prompt as string;
399
- const provider = (args.provider as string) || "grok";
400
- const imagePath = args.image as string | undefined;
401
- const output = (args.output as string) || `${provider}-${getTimestamp()}.mp4`;
402
- const duration = (args.duration as number) || (provider === "veo" ? 6 : 5);
403
-
404
- const runwayModel = (args.runwayModel as string) || "gen4.5";
405
-
406
- // Validate: Runway gen4_turbo requires an image; gen4.5 supports text-to-video
407
- if (provider === "runway" && !imagePath && runwayModel !== "gen4.5") {
408
- return {
409
- toolCallId: "",
410
- success: false,
411
- output: "",
412
- error: `Runway ${runwayModel} requires an input image. Provide 'image' parameter, use 'gen4.5' model, or use provider 'kling'/'veo' for text-to-video.`,
413
- };
414
- }
415
-
416
- // Helper to prepare reference image
417
- async function prepareReferenceImage(imgPath: string): Promise<string> {
418
- const absImagePath = resolve(context.workingDirectory, imgPath);
419
- const imageBuffer = await readFile(absImagePath);
420
- const base64 = imageBuffer.toString("base64");
421
- const ext = imgPath.split(".").pop()?.toLowerCase() || "png";
422
- const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : `image/${ext}`;
423
- return `data:${mimeType};base64,${base64}`;
424
- }
425
-
426
- try {
427
- if (provider === "runway") {
428
- const apiKey = await getApiKeyFromConfig("runway");
429
- if (!apiKey) {
430
- return {
431
- toolCallId: "",
432
- success: false,
433
- output: "",
434
- error: "Runway API key required. Configure via 'vibe setup'.",
435
- };
436
- }
437
-
438
- const { RunwayProvider } = await import("@vibeframe/ai-providers");
439
- const runway = new RunwayProvider();
440
- await runway.initialize({ apiKey });
441
-
442
- const referenceImage = imagePath ? await prepareReferenceImage(imagePath) : undefined;
443
-
444
- const result = await runway.generateVideo(prompt, {
445
- prompt,
446
- model: runwayModel,
447
- duration,
448
- referenceImage,
449
- });
450
-
451
- if (result.status === "failed") {
452
- return {
453
- toolCallId: "",
454
- success: false,
455
- output: "",
456
- error: `Video generation failed: ${result.error}`,
457
- };
458
- }
459
-
460
- if (result.status === "pending" || result.status === "processing") {
461
- let finalResult = result;
462
- const maxAttempts = 60;
463
- for (let i = 0; i < maxAttempts; i++) {
464
- await new Promise((r) => setTimeout(r, 5000));
465
- finalResult = await runway.getGenerationStatus(result.id);
466
- if (finalResult.status === "completed" || finalResult.status === "failed") {
467
- break;
468
- }
469
- }
470
-
471
- if (finalResult.status !== "completed") {
472
- return {
473
- toolCallId: "",
474
- success: false,
475
- output: "",
476
- error: `Video generation timed out or failed: ${finalResult.error || finalResult.status}`,
477
- };
478
- }
479
-
480
- if (finalResult.videoUrl) {
481
- const outputPath = resolve(context.workingDirectory, output);
482
- const buffer = await downloadVideo(finalResult.videoUrl, apiKey);
483
- await writeFile(outputPath, buffer);
484
- }
485
- }
486
-
487
- return {
488
- toolCallId: "",
489
- success: true,
490
- output: `Video generated: ${output}\nProvider: runway\nPrompt: ${prompt}\nDuration: ${duration}s`,
491
- };
492
- } else if (provider === "veo") {
493
- const apiKey = await getApiKeyFromConfig("google");
494
- if (!apiKey) {
495
- return {
496
- toolCallId: "",
497
- success: false,
498
- output: "",
499
- error: "Google API key required. Configure via 'vibe setup'.",
500
- };
501
- }
502
-
503
- const { GeminiProvider } = await import("@vibeframe/ai-providers");
504
- const gemini = new GeminiProvider();
505
- await gemini.initialize({ apiKey });
506
-
507
- const referenceImage = imagePath ? await prepareReferenceImage(imagePath) : undefined;
508
- const negativePrompt = args.negativePrompt as string | undefined;
509
- const resolution = args.resolution as "720p" | "1080p" | "4k" | undefined;
510
- const veoDuration = duration <= 6 ? (duration <= 4 ? 4 : 6) : 8;
511
-
512
- const result = await gemini.generateVideo(prompt, {
513
- prompt,
514
- duration: veoDuration as 4 | 6 | 8,
515
- referenceImage,
516
- model: "veo-3.1-fast-generate-preview",
517
- negativePrompt,
518
- resolution,
519
- });
520
-
521
- if (result.status === "failed") {
522
- return {
523
- toolCallId: "",
524
- success: false,
525
- output: "",
526
- error: `Veo video generation failed: ${result.error}`,
527
- };
528
- }
529
-
530
- if (result.status === "pending" || result.status === "processing") {
531
- const finalResult = await gemini.waitForVideoCompletion(
532
- result.id,
533
- undefined,
534
- 300000
535
- );
536
-
537
- if (finalResult.status !== "completed") {
538
- return {
539
- toolCallId: "",
540
- success: false,
541
- output: "",
542
- error: `Veo video generation timed out or failed: ${finalResult.error || finalResult.status}`,
543
- };
544
- }
545
-
546
- if (finalResult.videoUrl) {
547
- const outputPath = resolve(context.workingDirectory, output);
548
- const buffer = await downloadVideo(finalResult.videoUrl, apiKey);
549
- await writeFile(outputPath, buffer);
550
- }
551
- }
552
-
553
- return {
554
- toolCallId: "",
555
- success: true,
556
- output: `Video generated: ${output}\nProvider: veo\nPrompt: ${prompt}\nDuration: ${duration}s`,
557
- };
558
- } else if (provider === "grok") {
559
- const apiKey = await getApiKeyFromConfig("xai");
560
- if (!apiKey) {
561
- return {
562
- toolCallId: "",
563
- success: false,
564
- output: "",
565
- error: "xAI API key required. Configure via 'vibe setup'.",
566
- };
567
- }
568
-
569
- const { GrokProvider } = await import("@vibeframe/ai-providers");
570
- const grok = new GrokProvider();
571
- await grok.initialize({ apiKey });
572
-
573
- const referenceImage = imagePath ? await prepareReferenceImage(imagePath) : undefined;
574
-
575
- const result = await grok.generateVideo(prompt, {
576
- prompt,
577
- duration,
578
- referenceImage,
579
- aspectRatio: "16:9",
580
- });
581
-
582
- if (result.status === "failed") {
583
- return {
584
- toolCallId: "",
585
- success: false,
586
- output: "",
587
- error: `Grok video generation failed: ${result.error}`,
588
- };
589
- }
590
-
591
- if (result.status === "pending" || result.status === "processing") {
592
- const finalResult = await grok.waitForCompletion(
593
- result.id,
594
- undefined,
595
- 300000
596
- );
597
-
598
- if (finalResult.status !== "completed") {
599
- return {
600
- toolCallId: "",
601
- success: false,
602
- output: "",
603
- error: `Grok video generation timed out or failed: ${finalResult.error || finalResult.status}`,
604
- };
605
- }
606
-
607
- if (finalResult.videoUrl) {
608
- const outputPath = resolve(context.workingDirectory, output);
609
- const buffer = await downloadVideo(finalResult.videoUrl, apiKey);
610
- await writeFile(outputPath, buffer);
611
- }
612
- }
613
-
614
- return {
615
- toolCallId: "",
616
- success: true,
617
- output: `Video generated: ${output}\nProvider: grok (native audio)\nPrompt: ${prompt}\nDuration: ${duration}s`,
618
- };
619
- } else {
620
- // kling
621
- const apiKey = await getApiKeyFromConfig("kling");
622
- if (!apiKey) {
623
- return {
624
- toolCallId: "",
625
- success: false,
626
- output: "",
627
- error: "Kling API key required. Configure via 'vibe setup'.",
628
- };
629
- }
630
-
631
- const { KlingProvider } = await import("@vibeframe/ai-providers");
632
- const kling = new KlingProvider();
633
- await kling.initialize({ apiKey });
634
-
635
- let referenceImage = imagePath ? await prepareReferenceImage(imagePath) : undefined;
636
- const mode = (args.mode as "std" | "pro") || "std";
637
-
638
- // Kling v2.5+ requires image URL, not base64 — auto-upload to ImgBB
639
- if (referenceImage && referenceImage.startsWith("data:")) {
640
- const imgbbKey = await getApiKeyFromConfig("imgbb") || process.env.IMGBB_API_KEY;
641
- if (!imgbbKey) {
642
- return {
643
- toolCallId: "",
644
- success: false,
645
- output: "",
646
- error: "Kling requires image URL for image-to-video. Set IMGBB_API_KEY for auto-upload via 'vibe setup'.",
647
- };
648
- }
649
- const { uploadToImgbb } = await import("../../commands/ai-script-pipeline.js");
650
- const base64Data = referenceImage.split(",")[1];
651
- const imageBuffer = Buffer.from(base64Data, "base64");
652
- const uploadResult = await uploadToImgbb(imageBuffer, imgbbKey);
653
- if (!uploadResult.success || !uploadResult.url) {
654
- return {
655
- toolCallId: "",
656
- success: false,
657
- output: "",
658
- error: `ImgBB upload failed: ${uploadResult.error || "Unknown error"}`,
659
- };
660
- }
661
- referenceImage = uploadResult.url;
662
- }
663
-
664
- const result = await kling.generateVideo(prompt, {
665
- prompt,
666
- duration: duration as 5 | 10,
667
- mode,
668
- referenceImage,
669
- });
670
-
671
- if (result.status === "failed") {
672
- return {
673
- toolCallId: "",
674
- success: false,
675
- output: "",
676
- error: `Kling video generation failed: ${result.error}`,
677
- };
678
- }
679
-
680
- if (result.status === "pending" || result.status === "processing") {
681
- let finalResult = result;
682
- const maxAttempts = 60;
683
- for (let i = 0; i < maxAttempts; i++) {
684
- await new Promise((r) => setTimeout(r, 5000));
685
- finalResult = await kling.getGenerationStatus(result.id);
686
- if (finalResult.status === "completed" || finalResult.status === "failed") {
687
- break;
688
- }
689
- }
690
-
691
- if (finalResult.status !== "completed") {
692
- return {
693
- toolCallId: "",
694
- success: false,
695
- output: "",
696
- error: `Kling video generation timed out or failed: ${finalResult.error || finalResult.status}`,
697
- };
698
- }
699
-
700
- if (finalResult.videoUrl) {
701
- const outputPath = resolve(context.workingDirectory, output);
702
- const buffer = await downloadVideo(finalResult.videoUrl, apiKey);
703
- await writeFile(outputPath, buffer);
704
- }
705
- }
706
-
707
- return {
708
- toolCallId: "",
709
- success: true,
710
- output: `Video generated: ${output}\nProvider: kling\nPrompt: ${prompt}\nDuration: ${duration}s\nMode: ${mode}`,
711
- };
712
- }
713
- } catch (error) {
714
- return {
715
- toolCallId: "",
716
- success: false,
717
- output: "",
718
- error: `Failed to generate video: ${error instanceof Error ? error.message : String(error)}`,
719
- };
720
- }
721
- };
722
-
723
- const generateTTS: ToolHandler = async (args, context): Promise<ToolResult> => {
724
- const text = args.text as string;
725
- const output = (args.output as string) || `tts-${getTimestamp()}.mp3`;
726
- const voice = args.voice as string | undefined;
727
-
728
- try {
729
- const apiKey = await getApiKeyFromConfig("elevenlabs");
730
- if (!apiKey) {
731
- return {
732
- toolCallId: "",
733
- success: false,
734
- output: "",
735
- error: "ElevenLabs API key required. Configure via 'vibe setup'.",
736
- };
737
- }
738
-
739
- const { ElevenLabsProvider } = await import("@vibeframe/ai-providers");
740
- const elevenlabs = new ElevenLabsProvider();
741
- await elevenlabs.initialize({ apiKey });
742
-
743
- const result = await elevenlabs.textToSpeech(text, {
744
- voiceId: voice,
745
- });
746
-
747
- if (!result.success || !result.audioBuffer) {
748
- return {
749
- toolCallId: "",
750
- success: false,
751
- output: "",
752
- error: `TTS generation failed: ${result.error || "No audio generated"}`,
753
- };
754
- }
755
-
756
- // Save audio
757
- const outputPath = resolve(context.workingDirectory, output);
758
- await writeFile(outputPath, result.audioBuffer);
759
-
760
- return {
761
- toolCallId: "",
762
- success: true,
763
- output: `Speech generated: ${output}\nText: ${text.substring(0, 100)}${text.length > 100 ? "..." : ""}`,
764
- };
765
- } catch (error) {
766
- return {
767
- toolCallId: "",
768
- success: false,
769
- output: "",
770
- error: `Failed to generate speech: ${error instanceof Error ? error.message : String(error)}`,
771
- };
772
- }
773
- };
774
-
775
- const generateSFX: ToolHandler = async (args, context): Promise<ToolResult> => {
776
- const prompt = args.prompt as string;
777
- const output = (args.output as string) || `sfx-${getTimestamp()}.mp3`;
778
- const duration = args.duration as number | undefined;
779
-
780
- try {
781
- const apiKey = await getApiKeyFromConfig("elevenlabs");
782
- if (!apiKey) {
783
- return {
784
- toolCallId: "",
785
- success: false,
786
- output: "",
787
- error: "ElevenLabs API key required. Configure via 'vibe setup'.",
788
- };
789
- }
790
-
791
- const { ElevenLabsProvider } = await import("@vibeframe/ai-providers");
792
- const elevenlabs = new ElevenLabsProvider();
793
- await elevenlabs.initialize({ apiKey });
794
-
795
- const result = await elevenlabs.generateSoundEffect(prompt, {
796
- duration,
797
- });
798
-
799
- if (!result.success || !result.audioBuffer) {
800
- return {
801
- toolCallId: "",
802
- success: false,
803
- output: "",
804
- error: `SFX generation failed: ${result.error || "No audio generated"}`,
805
- };
806
- }
807
-
808
- // Save audio
809
- const outputPath = resolve(context.workingDirectory, output);
810
- await writeFile(outputPath, result.audioBuffer);
811
-
812
- return {
813
- toolCallId: "",
814
- success: true,
815
- output: `Sound effect generated: ${output}\nPrompt: ${prompt}`,
816
- };
817
- } catch (error) {
818
- return {
819
- toolCallId: "",
820
- success: false,
821
- output: "",
822
- error: `Failed to generate sound effect: ${error instanceof Error ? error.message : String(error)}`,
823
- };
824
- }
825
- };
826
-
827
- const generateMusic: ToolHandler = async (args, context): Promise<ToolResult> => {
828
- const prompt = args.prompt as string;
829
- const output = (args.output as string) || `music-${getTimestamp()}.mp3`;
830
- const duration = (args.duration as number) || 8;
831
- const provider = (args.provider as string) || "elevenlabs";
832
- const instrumental = (args.instrumental as boolean) || false;
833
-
834
- try {
835
- if (provider === "elevenlabs") {
836
- // ElevenLabs Music API — synchronous, up to 10 minutes
837
- const apiKey = await getApiKeyFromConfig("elevenlabs");
838
- if (!apiKey) {
839
- return {
840
- toolCallId: "",
841
- success: false,
842
- output: "",
843
- error: "ElevenLabs API key required. Configure via 'vibe setup'.",
844
- };
845
- }
846
-
847
- const { ElevenLabsProvider } = await import("@vibeframe/ai-providers");
848
- const elevenlabs = new ElevenLabsProvider();
849
- await elevenlabs.initialize({ apiKey });
850
-
851
- const result = await elevenlabs.generateMusic(prompt, {
852
- duration,
853
- forceInstrumental: instrumental,
854
- });
855
-
856
- if (!result.success || !result.audioBuffer) {
857
- return {
858
- toolCallId: "",
859
- success: false,
860
- output: "",
861
- error: `Music generation failed: ${result.error || "No audio generated"}`,
862
- };
863
- }
864
-
865
- const outputPath = resolve(context.workingDirectory, output);
866
- await writeFile(outputPath, result.audioBuffer);
867
-
868
- return {
869
- toolCallId: "",
870
- success: true,
871
- output: `Music generated: ${output}\nPrompt: ${prompt}\nDuration: ${duration}s\nProvider: ElevenLabs${instrumental ? "\nMode: Instrumental" : ""}`,
872
- };
873
- } else {
874
- // Replicate MusicGen — async polling
875
- const apiKey = await getApiKeyFromConfig("replicate");
876
- if (!apiKey) {
877
- return {
878
- toolCallId: "",
879
- success: false,
880
- output: "",
881
- error: "Replicate API key required. Configure via 'vibe setup'.",
882
- };
883
- }
884
-
885
- const { ReplicateProvider } = await import("@vibeframe/ai-providers");
886
- const replicate = new ReplicateProvider();
887
- await replicate.initialize({ apiKey });
888
-
889
- const result = await replicate.generateMusic(prompt, {
890
- duration,
891
- });
892
-
893
- if (!result.success) {
894
- return {
895
- toolCallId: "",
896
- success: false,
897
- output: "",
898
- error: `Music generation failed: ${result.error || "Unknown error"}`,
899
- };
900
- }
901
-
902
- // Music generation is async - need to poll
903
- if (result.taskId) {
904
- let finalResult = result;
905
- const maxAttempts = 60;
906
- for (let i = 0; i < maxAttempts; i++) {
907
- await new Promise((r) => setTimeout(r, 3000));
908
- finalResult = await replicate.getMusicStatus(result.taskId);
909
- if (finalResult.success && finalResult.audioUrl) {
910
- break;
911
- }
912
- if (finalResult.error) {
913
- return {
914
- toolCallId: "",
915
- success: false,
916
- output: "",
917
- error: `Music generation failed: ${finalResult.error}`,
918
- };
919
- }
920
- }
921
-
922
- if (finalResult.audioUrl) {
923
- const outputPath = resolve(context.workingDirectory, output);
924
- const response = await fetch(finalResult.audioUrl);
925
- const buffer = Buffer.from(await response.arrayBuffer());
926
- await writeFile(outputPath, buffer);
927
- } else {
928
- return {
929
- toolCallId: "",
930
- success: false,
931
- output: "",
932
- error: "Music generation timed out",
933
- };
934
- }
935
- }
936
-
937
- return {
938
- toolCallId: "",
939
- success: true,
940
- output: `Music generated: ${output}\nPrompt: ${prompt}\nDuration: ${duration}s\nProvider: Replicate`,
941
- };
942
- }
943
- } catch (error) {
944
- return {
945
- toolCallId: "",
946
- success: false,
947
- output: "",
948
- error: `Failed to generate music: ${error instanceof Error ? error.message : String(error)}`,
949
- };
950
- }
951
- };
952
-
953
- const generateStoryboard: ToolHandler = async (args, context): Promise<ToolResult> => {
954
- const script = args.script as string;
955
- const targetDuration = args.targetDuration as number | undefined;
956
- const output = (args.output as string) || `storyboard-${getTimestamp()}.json`;
957
- const creativity = args.creativity as "low" | "high" | undefined;
958
-
959
- try {
960
- const apiKey = await getApiKeyFromConfig("anthropic");
961
- if (!apiKey) {
962
- return {
963
- toolCallId: "",
964
- success: false,
965
- output: "",
966
- error: "Anthropic API key required. Configure via 'vibe setup'.",
967
- };
968
- }
969
-
970
- const { ClaudeProvider } = await import("@vibeframe/ai-providers");
971
- const claude = new ClaudeProvider();
972
- await claude.initialize({ apiKey });
973
-
974
- const result = await claude.analyzeContent(script, targetDuration, { creativity });
975
-
976
- if (!result || result.length === 0) {
977
- return {
978
- toolCallId: "",
979
- success: false,
980
- output: "",
981
- error: "Failed to generate storyboard",
982
- };
983
- }
984
-
985
- // Save storyboard
986
- const outputPath = resolve(context.workingDirectory, output);
987
- await writeFile(outputPath, JSON.stringify(result, null, 2), "utf-8");
988
-
989
- // Format summary
990
- const sanitizedResult = sanitizeAIResult(result);
991
- const summary = sanitizedResult.map((scene, i) =>
992
- `Scene ${i + 1}: ${scene.description.substring(0, 60)}...`
993
- ).join("\n");
994
-
995
- return {
996
- toolCallId: "",
997
- success: true,
998
- output: `Storyboard generated: ${output}\n\n${summary}`,
999
- };
1000
- } catch (error) {
1001
- return {
1002
- toolCallId: "",
1003
- success: false,
1004
- output: "",
1005
- error: `Failed to generate storyboard: ${error instanceof Error ? error.message : String(error)}`,
1006
- };
1007
- }
1008
- };
1009
-
1010
- const generateMotion: ToolHandler = async (args, context): Promise<ToolResult> => {
1011
- try {
1012
- const { executeMotion } = await import("../../commands/ai-motion.js");
1013
-
1014
- const video = args.video
1015
- ? resolve(context.workingDirectory, args.video as string)
1016
- : undefined;
1017
- const output = args.output
1018
- ? resolve(context.workingDirectory, args.output as string)
1019
- : undefined;
1020
-
1021
- const result = await executeMotion({
1022
- description: args.description as string,
1023
- duration: args.duration as number | undefined,
1024
- width: args.width as number | undefined,
1025
- height: args.height as number | undefined,
1026
- style: args.style as string | undefined,
1027
- render: true, // Always render in agent mode
1028
- video,
1029
- output,
1030
- });
1031
-
1032
- if (!result.success) {
1033
- return {
1034
- toolCallId: "",
1035
- success: false,
1036
- output: result.codePath ? `TSX code saved to: ${result.codePath}` : "",
1037
- error: result.error || "Motion generation failed",
1038
- };
1039
- }
1040
-
1041
- const parts: string[] = [];
1042
- if (result.codePath) parts.push(`Code: ${result.codePath}`);
1043
- if (result.renderedPath) parts.push(`Rendered: ${result.renderedPath}`);
1044
- if (result.compositedPath) parts.push(`Composited: ${result.compositedPath}`);
1045
-
1046
- return {
1047
- toolCallId: "",
1048
- success: true,
1049
- output: parts.join("\n"),
1050
- };
1051
- } catch (error) {
1052
- return {
1053
- toolCallId: "",
1054
- success: false,
1055
- output: "",
1056
- error: `Motion generation failed: ${error instanceof Error ? error.message : String(error)}`,
1057
- };
1058
- }
1059
- };
1060
-
1061
- // ============================================================================
1062
- // Registration
1063
- // ============================================================================
1064
-
1065
- export function registerGenerationTools(registry: ToolRegistry): void {
1066
- registry.register(imageDef, generateImage);
1067
- registry.register(videoDef, generateVideo);
1068
- registry.register(ttsDef, generateTTS);
1069
- registry.register(sfxDef, generateSFX);
1070
- registry.register(musicDef, generateMusic);
1071
- registry.register(storyboardDef, generateStoryboard);
1072
- registry.register(motionDef, generateMotion);
1073
- }