@vibeframe/cli 0.27.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/LICENSE +21 -0
  2. package/dist/agent/adapters/index.d.ts +1 -0
  3. package/dist/agent/adapters/index.d.ts.map +1 -1
  4. package/dist/agent/adapters/index.js +5 -0
  5. package/dist/agent/adapters/index.js.map +1 -1
  6. package/dist/agent/adapters/openrouter.d.ts +16 -0
  7. package/dist/agent/adapters/openrouter.d.ts.map +1 -0
  8. package/dist/agent/adapters/openrouter.js +100 -0
  9. package/dist/agent/adapters/openrouter.js.map +1 -0
  10. package/dist/agent/types.d.ts +1 -1
  11. package/dist/agent/types.d.ts.map +1 -1
  12. package/dist/commands/agent.d.ts.map +1 -1
  13. package/dist/commands/agent.js +3 -1
  14. package/dist/commands/agent.js.map +1 -1
  15. package/dist/commands/setup.js +5 -2
  16. package/dist/commands/setup.js.map +1 -1
  17. package/dist/config/schema.d.ts +2 -1
  18. package/dist/config/schema.d.ts.map +1 -1
  19. package/dist/config/schema.js +2 -0
  20. package/dist/config/schema.js.map +1 -1
  21. package/dist/index.js +0 -0
  22. package/package.json +16 -12
  23. package/.turbo/turbo-build.log +0 -4
  24. package/.turbo/turbo-lint.log +0 -21
  25. package/.turbo/turbo-test.log +0 -689
  26. package/src/agent/adapters/claude.ts +0 -143
  27. package/src/agent/adapters/gemini.ts +0 -159
  28. package/src/agent/adapters/index.ts +0 -61
  29. package/src/agent/adapters/ollama.ts +0 -231
  30. package/src/agent/adapters/openai.ts +0 -116
  31. package/src/agent/adapters/xai.ts +0 -119
  32. package/src/agent/index.ts +0 -251
  33. package/src/agent/memory/index.ts +0 -151
  34. package/src/agent/prompts/system.ts +0 -106
  35. package/src/agent/tools/ai-editing.ts +0 -845
  36. package/src/agent/tools/ai-generation.ts +0 -1073
  37. package/src/agent/tools/ai-pipeline.ts +0 -1055
  38. package/src/agent/tools/ai.ts +0 -21
  39. package/src/agent/tools/batch.ts +0 -429
  40. package/src/agent/tools/e2e.test.ts +0 -545
  41. package/src/agent/tools/export.ts +0 -184
  42. package/src/agent/tools/filesystem.ts +0 -237
  43. package/src/agent/tools/index.ts +0 -150
  44. package/src/agent/tools/integration.test.ts +0 -775
  45. package/src/agent/tools/media.ts +0 -697
  46. package/src/agent/tools/project.ts +0 -313
  47. package/src/agent/tools/timeline.ts +0 -951
  48. package/src/agent/types.ts +0 -68
  49. package/src/commands/agent.ts +0 -340
  50. package/src/commands/ai-analyze.ts +0 -429
  51. package/src/commands/ai-animated-caption.ts +0 -390
  52. package/src/commands/ai-audio.ts +0 -941
  53. package/src/commands/ai-broll.ts +0 -490
  54. package/src/commands/ai-edit-cli.ts +0 -658
  55. package/src/commands/ai-edit.ts +0 -1542
  56. package/src/commands/ai-fill-gaps.ts +0 -566
  57. package/src/commands/ai-helpers.ts +0 -65
  58. package/src/commands/ai-highlights.ts +0 -1303
  59. package/src/commands/ai-image.ts +0 -761
  60. package/src/commands/ai-motion.ts +0 -347
  61. package/src/commands/ai-narrate.ts +0 -451
  62. package/src/commands/ai-review.ts +0 -309
  63. package/src/commands/ai-script-pipeline-cli.ts +0 -1710
  64. package/src/commands/ai-script-pipeline.ts +0 -1365
  65. package/src/commands/ai-suggest-edit.ts +0 -264
  66. package/src/commands/ai-video-fx.ts +0 -445
  67. package/src/commands/ai-video.ts +0 -915
  68. package/src/commands/ai-viral.ts +0 -595
  69. package/src/commands/ai-visual-fx.ts +0 -601
  70. package/src/commands/ai.test.ts +0 -627
  71. package/src/commands/ai.ts +0 -307
  72. package/src/commands/analyze.ts +0 -282
  73. package/src/commands/audio.ts +0 -644
  74. package/src/commands/batch.test.ts +0 -279
  75. package/src/commands/batch.ts +0 -440
  76. package/src/commands/detect.ts +0 -329
  77. package/src/commands/doctor.ts +0 -237
  78. package/src/commands/edit-cmd.ts +0 -1014
  79. package/src/commands/export.ts +0 -918
  80. package/src/commands/generate.ts +0 -2146
  81. package/src/commands/media.ts +0 -177
  82. package/src/commands/output.ts +0 -142
  83. package/src/commands/pipeline.ts +0 -398
  84. package/src/commands/project.test.ts +0 -127
  85. package/src/commands/project.ts +0 -149
  86. package/src/commands/sanitize.ts +0 -60
  87. package/src/commands/schema.ts +0 -130
  88. package/src/commands/setup.ts +0 -509
  89. package/src/commands/timeline.test.ts +0 -499
  90. package/src/commands/timeline.ts +0 -529
  91. package/src/commands/validate.ts +0 -77
  92. package/src/config/config.test.ts +0 -197
  93. package/src/config/index.ts +0 -125
  94. package/src/config/schema.ts +0 -82
  95. package/src/engine/index.ts +0 -2
  96. package/src/engine/project.test.ts +0 -702
  97. package/src/engine/project.ts +0 -439
  98. package/src/index.ts +0 -146
  99. package/src/utils/api-key.test.ts +0 -41
  100. package/src/utils/api-key.ts +0 -247
  101. package/src/utils/audio.ts +0 -83
  102. package/src/utils/exec-safe.ts +0 -75
  103. package/src/utils/first-run.ts +0 -52
  104. package/src/utils/provider-resolver.ts +0 -56
  105. package/src/utils/remotion.ts +0 -951
  106. package/src/utils/subtitle.test.ts +0 -227
  107. package/src/utils/subtitle.ts +0 -169
  108. package/src/utils/tty.ts +0 -196
  109. package/tsconfig.json +0 -20
@@ -1,1710 +0,0 @@
1
- /**
2
- * @module ai-script-pipeline-cli
3
- * @description CLI command registration for the script-to-video pipeline and
4
- * scene regeneration commands. Execute functions and helpers live in
5
- * ai-script-pipeline.ts; this file wires them up as Commander.js subcommands.
6
- */
7
-
8
- import { Command } from "commander";
9
- import { readFile, writeFile, mkdir, stat } from "node:fs/promises";
10
- import { resolve, dirname, extname } from "node:path";
11
- import { existsSync } from "node:fs";
12
- import chalk from "chalk";
13
- import ora from "ora";
14
- import {
15
- GeminiProvider,
16
- OpenAIProvider,
17
- OpenAIImageProvider,
18
- ClaudeProvider,
19
- ElevenLabsProvider,
20
- KlingProvider,
21
- RunwayProvider,
22
- GrokProvider,
23
- } from "@vibeframe/ai-providers";
24
- import { getApiKey, loadEnv } from "../utils/api-key.js";
25
- import { getApiKeyFromConfig } from "../config/index.js";
26
- import { Project, type ProjectFile } from "../engine/index.js";
27
- import { getAudioDuration } from "../utils/audio.js";
28
- import { applyTextOverlays, type TextOverlayStyle } from "./ai-edit.js";
29
- import { executeReview } from "./ai-review.js";
30
- import {
31
- type StoryboardSegment,
32
- DEFAULT_VIDEO_RETRIES,
33
- RETRY_DELAY_MS,
34
- sleep,
35
- uploadToImgbb,
36
- extendVideoToTarget,
37
- generateVideoWithRetryKling,
38
- generateVideoWithRetryRunway,
39
- } from "./ai-script-pipeline.js";
40
- import { downloadVideo } from "./ai-helpers.js";
41
-
42
- export function registerScriptPipelineCommands(aiCommand: Command): void {
43
- // Script-to-Video command
44
- aiCommand
45
- .command("script-to-video")
46
- .alias("s2v")
47
- .description("Generate complete video from text script using AI pipeline")
48
- .argument("<script>", "Script text or file path (use -f for file)")
49
- .option("-f, --file", "Treat script argument as file path")
50
- .option("-o, --output <path>", "Output project file path", "script-video.vibe.json")
51
- .option("-d, --duration <seconds>", "Target total duration in seconds")
52
- .option("-v, --voice <id>", "ElevenLabs voice ID for narration")
53
- .option("-g, --generator <engine>", "Video generator: kling | runway | veo", "kling")
54
- .option("-i, --image-provider <provider>", "Image provider: gemini | openai | grok", "gemini")
55
- .option("-a, --aspect-ratio <ratio>", "Aspect ratio: 16:9 | 9:16 | 1:1", "16:9")
56
- .option("--images-only", "Generate images only, skip video generation")
57
- .option("--no-voiceover", "Skip voiceover generation")
58
- .option("--output-dir <dir>", "Directory for generated assets", "script-video-output")
59
- .option("--retries <count>", "Number of retries for video generation failures", String(DEFAULT_VIDEO_RETRIES))
60
- .option("--sequential", "Generate videos one at a time (slower but more reliable)")
61
- .option("--concurrency <count>", "Max concurrent video tasks in parallel mode (default: 3)", "3")
62
- .option("-c, --creativity <level>", "Creativity level: low (default, consistent) or high (varied, unexpected)", "low")
63
- .option("-s, --storyboard-provider <provider>", "Storyboard provider: claude (default), openai, or gemini", "claude")
64
- .option("--no-text-overlay", "Skip text overlay step")
65
- .option("--text-style <style>", "Text overlay style: lower-third, center-bold, subtitle, minimal", "lower-third")
66
- .option("--review", "Run AI review after assembly (requires GOOGLE_API_KEY)")
67
- .option("--review-auto-apply", "Auto-apply fixable issues from AI review")
68
- .action(async (script: string, options) => {
69
- try {
70
- // Load environment variables from .env file
71
- loadEnv();
72
-
73
- // Get storyboard provider API key
74
- const storyboardProvider = (options.storyboardProvider || "claude") as "claude" | "openai" | "gemini";
75
- let storyboardApiKey: string | undefined;
76
-
77
- if (storyboardProvider === "openai") {
78
- storyboardApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
79
- if (!storyboardApiKey) {
80
- console.error(chalk.red("OpenAI API key required for storyboard generation (-s openai). Set OPENAI_API_KEY in .env or run: vibe setup"));
81
- process.exit(1);
82
- }
83
- } else if (storyboardProvider === "gemini") {
84
- storyboardApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
85
- if (!storyboardApiKey) {
86
- console.error(chalk.red("Google API key required for storyboard generation (-s gemini). Set GOOGLE_API_KEY in .env or run: vibe setup"));
87
- process.exit(1);
88
- }
89
- } else if (storyboardProvider === "claude") {
90
- storyboardApiKey = (await getApiKey("ANTHROPIC_API_KEY", "Anthropic")) ?? undefined;
91
- if (!storyboardApiKey) {
92
- console.error(chalk.red("Anthropic API key required for storyboard generation. Set ANTHROPIC_API_KEY in .env or run: vibe setup"));
93
- process.exit(1);
94
- }
95
- } else {
96
- console.error(chalk.red(`Unknown storyboard provider: ${storyboardProvider}. Use claude, openai, or gemini`));
97
- process.exit(1);
98
- }
99
-
100
- // Get image provider API key
101
- let imageApiKey: string | undefined;
102
- const imageProvider = options.imageProvider || "openai";
103
-
104
- if (imageProvider === "openai" || imageProvider === "dalle") {
105
- imageApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
106
- if (!imageApiKey) {
107
- console.error(chalk.red("OpenAI API key required for DALL-E image generation. Set OPENAI_API_KEY in .env or run: vibe setup"));
108
- process.exit(1);
109
- }
110
- } else if (imageProvider === "gemini") {
111
- imageApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
112
- if (!imageApiKey) {
113
- console.error(chalk.red("Google API key required for Gemini image generation. Set GOOGLE_API_KEY in .env or run: vibe setup"));
114
- process.exit(1);
115
- }
116
- } else if (imageProvider === "grok") {
117
- imageApiKey = (await getApiKey("XAI_API_KEY", "xAI")) ?? undefined;
118
- if (!imageApiKey) {
119
- console.error(chalk.red("xAI API key required for Grok image generation. Set XAI_API_KEY in .env or run: vibe setup"));
120
- process.exit(1);
121
- }
122
- } else {
123
- console.error(chalk.red(`Unknown image provider: ${imageProvider}. Use openai, gemini, or grok`));
124
- process.exit(1);
125
- }
126
-
127
- let elevenlabsApiKey: string | undefined;
128
- if (options.voiceover !== false) {
129
- const key = await getApiKey("ELEVENLABS_API_KEY", "ElevenLabs");
130
- if (!key) {
131
- console.error(chalk.red("ElevenLabs API key required for voiceover (or use --no-voiceover). Set ELEVENLABS_API_KEY in .env or run: vibe setup"));
132
- process.exit(1);
133
- }
134
- elevenlabsApiKey = key;
135
- }
136
-
137
- let videoApiKey: string | undefined;
138
- if (!options.imagesOnly) {
139
- if (options.generator === "kling") {
140
- const key = await getApiKey("KLING_API_KEY", "Kling");
141
- if (!key) {
142
- console.error(chalk.red("Kling API key required (or use --images-only). Set KLING_API_KEY in .env or run: vibe setup"));
143
- process.exit(1);
144
- }
145
- videoApiKey = key;
146
- } else {
147
- const key = await getApiKey("RUNWAY_API_SECRET", "Runway");
148
- if (!key) {
149
- console.error(chalk.red("Runway API key required (or use --images-only). Set RUNWAY_API_SECRET in .env or run: vibe setup"));
150
- process.exit(1);
151
- }
152
- videoApiKey = key;
153
- }
154
- }
155
-
156
- // Read script content
157
- let scriptContent = script;
158
- if (options.file) {
159
- const filePath = resolve(process.cwd(), script);
160
- scriptContent = await readFile(filePath, "utf-8");
161
- }
162
-
163
- // Determine output directory for assets
164
- // If -o looks like a directory and --output-dir is not explicitly set, use -o directory for assets
165
- let effectiveOutputDir = options.outputDir;
166
- const outputLooksLikeDirectory =
167
- options.output.endsWith("/") ||
168
- (!options.output.endsWith(".json") && !options.output.endsWith(".vibe.json"));
169
-
170
- if (outputLooksLikeDirectory && options.outputDir === "script-video-output") {
171
- // User specified a directory for -o but didn't set --output-dir, use -o directory for assets
172
- effectiveOutputDir = options.output;
173
- }
174
-
175
- // Create output directory
176
- const outputDir = resolve(process.cwd(), effectiveOutputDir);
177
- if (!existsSync(outputDir)) {
178
- await mkdir(outputDir, { recursive: true });
179
- }
180
-
181
- // Validate creativity level
182
- const creativity = options.creativity?.toLowerCase();
183
- if (creativity && creativity !== "low" && creativity !== "high") {
184
- console.error(chalk.red("Invalid creativity level. Use 'low' or 'high'."));
185
- process.exit(1);
186
- }
187
-
188
- console.log();
189
- console.log(chalk.bold.cyan("🎬 Script-to-Video Pipeline"));
190
- console.log(chalk.dim("─".repeat(60)));
191
- if (creativity === "high") {
192
- console.log(chalk.yellow("🎨 High creativity mode: Generating varied, unexpected scenes"));
193
- }
194
- console.log();
195
-
196
- // Step 1: Generate storyboard
197
- const providerLabel = storyboardProvider.charAt(0).toUpperCase() + storyboardProvider.slice(1);
198
- const storyboardSpinnerText = creativity === "high"
199
- ? `Analyzing script with ${providerLabel} (high creativity)...`
200
- : `Analyzing script with ${providerLabel}...`;
201
- const storyboardSpinner = ora(storyboardSpinnerText).start();
202
-
203
- let segments: StoryboardSegment[];
204
- const creativityOpts = { creativity: creativity as "low" | "high" | undefined };
205
- const durationOpt = options.duration ? parseFloat(options.duration) : undefined;
206
-
207
- if (storyboardProvider === "openai") {
208
- const openai = new OpenAIProvider();
209
- await openai.initialize({ apiKey: storyboardApiKey! });
210
- segments = await openai.analyzeContent(scriptContent, durationOpt, creativityOpts);
211
- } else if (storyboardProvider === "gemini") {
212
- const gemini = new GeminiProvider();
213
- await gemini.initialize({ apiKey: storyboardApiKey! });
214
- segments = await gemini.analyzeContent(scriptContent, durationOpt, creativityOpts);
215
- } else {
216
- const claude = new ClaudeProvider();
217
- await claude.initialize({ apiKey: storyboardApiKey! });
218
- segments = await claude.analyzeContent(scriptContent, durationOpt, creativityOpts);
219
- }
220
-
221
- if (segments.length === 0) {
222
- storyboardSpinner.fail(chalk.red("Failed to generate storyboard (check API key and error above)"));
223
- process.exit(1);
224
- }
225
-
226
- let totalDuration = segments.reduce((sum, seg) => sum + seg.duration, 0);
227
- storyboardSpinner.succeed(chalk.green(`Generated ${segments.length} scenes (total: ${totalDuration}s)`));
228
-
229
- // Save storyboard
230
- const storyboardPath = resolve(outputDir, "storyboard.json");
231
- await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
232
- console.log(chalk.dim(` → Saved: ${storyboardPath}`));
233
- console.log();
234
-
235
- // Step 2: Generate per-scene voiceovers with ElevenLabs
236
- const perSceneTTS: { path: string; duration: number; segmentIndex: number }[] = [];
237
- const failedNarrations: { sceneNum: number; error: string }[] = [];
238
-
239
- if (options.voiceover !== false && elevenlabsApiKey) {
240
- const ttsSpinner = ora("🎙️ Generating voiceovers with ElevenLabs...").start();
241
-
242
- const elevenlabs = new ElevenLabsProvider();
243
- await elevenlabs.initialize({ apiKey: elevenlabsApiKey });
244
-
245
- let totalCharacters = 0;
246
-
247
- for (let i = 0; i < segments.length; i++) {
248
- const segment = segments[i];
249
- const narrationText = segment.narration || segment.description;
250
-
251
- if (!narrationText) continue;
252
-
253
- ttsSpinner.text = `🎙️ Generating narration ${i + 1}/${segments.length}...`;
254
-
255
- let ttsResult = await elevenlabs.textToSpeech(narrationText, {
256
- voiceId: options.voice,
257
- });
258
-
259
- if (!ttsResult.success || !ttsResult.audioBuffer) {
260
- const errorMsg = ttsResult.error || "Unknown error";
261
- failedNarrations.push({ sceneNum: i + 1, error: errorMsg });
262
- ttsSpinner.text = `🎙️ Generating narration ${i + 1}/${segments.length}... (failed)`;
263
- console.log(chalk.yellow(`\n ⚠ Narration ${i + 1} failed: ${errorMsg}`));
264
- continue;
265
- }
266
-
267
- const audioPath = resolve(outputDir, `narration-${i + 1}.mp3`);
268
- await writeFile(audioPath, ttsResult.audioBuffer);
269
-
270
- // Get actual audio duration using ffprobe
271
- let actualDuration = await getAudioDuration(audioPath);
272
-
273
- // Auto speed-adjust if narration slightly exceeds video bracket (5s or 10s)
274
- const videoBracket = segment.duration > 5 ? 10 : 5;
275
- const overageRatio = actualDuration / videoBracket;
276
- if (overageRatio > 1.0 && overageRatio <= 1.15) {
277
- // Narration exceeds bracket by 0-15% — regenerate slightly faster
278
- const adjustedSpeed = Math.min(1.2, parseFloat(overageRatio.toFixed(2)));
279
- ttsSpinner.text = `🎙️ Narration ${i + 1}: adjusting speed to ${adjustedSpeed}x...`;
280
- const speedResult = await elevenlabs.textToSpeech(narrationText, {
281
- voiceId: options.voice,
282
- speed: adjustedSpeed,
283
- });
284
- if (speedResult.success && speedResult.audioBuffer) {
285
- await writeFile(audioPath, speedResult.audioBuffer);
286
- actualDuration = await getAudioDuration(audioPath);
287
- ttsResult = speedResult;
288
- console.log(chalk.dim(` → Speed-adjusted narration ${i + 1}: ${adjustedSpeed}x → ${actualDuration.toFixed(1)}s`));
289
- }
290
- }
291
-
292
- // Update segment duration to match actual narration length
293
- segment.duration = actualDuration;
294
-
295
- perSceneTTS.push({ path: audioPath, duration: actualDuration, segmentIndex: i });
296
- totalCharacters += ttsResult.characterCount || 0;
297
-
298
- console.log(chalk.dim(` → Saved: ${audioPath} (${actualDuration.toFixed(1)}s)`));
299
- }
300
-
301
- // Recalculate startTime for all segments based on updated durations
302
- let currentTime = 0;
303
- for (const segment of segments) {
304
- segment.startTime = currentTime;
305
- currentTime += segment.duration;
306
- }
307
-
308
- // Update total duration
309
- totalDuration = segments.reduce((sum, seg) => sum + seg.duration, 0);
310
-
311
- // Show success with failed count if any
312
- if (failedNarrations.length > 0) {
313
- ttsSpinner.warn(chalk.yellow(`Generated ${perSceneTTS.length}/${segments.length} narrations (${failedNarrations.length} failed)`));
314
- } else {
315
- ttsSpinner.succeed(chalk.green(`Generated ${perSceneTTS.length}/${segments.length} narrations (${totalCharacters} chars, ${totalDuration.toFixed(1)}s total)`));
316
- }
317
-
318
- // Re-save storyboard with updated durations
319
- await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
320
- console.log(chalk.dim(` → Updated storyboard: ${storyboardPath}`));
321
- console.log();
322
- }
323
-
324
- // Step 3: Generate images with selected provider
325
- const providerNames: Record<string, string> = {
326
- openai: "OpenAI GPT Image 1.5",
327
- dalle: "OpenAI GPT Image 1.5", // backward compatibility
328
- gemini: "Gemini",
329
- grok: "xAI Grok",
330
- };
331
- const imageSpinner = ora(`🎨 Generating visuals with ${providerNames[imageProvider]}...`).start();
332
-
333
- // Determine image size/aspect ratio based on provider
334
- const dalleImageSizes: Record<string, "1536x1024" | "1024x1536" | "1024x1024"> = {
335
- "16:9": "1536x1024",
336
- "9:16": "1024x1536",
337
- "1:1": "1024x1024",
338
- };
339
- const imagePaths: string[] = [];
340
-
341
- // Store first scene image for style continuity
342
- let firstSceneImage: Buffer | undefined;
343
-
344
- // Initialize the selected provider
345
- let openaiImageInstance: OpenAIImageProvider | undefined;
346
- let geminiInstance: GeminiProvider | undefined;
347
- let grokInstance: GrokProvider | undefined;
348
-
349
- if (imageProvider === "openai" || imageProvider === "dalle") {
350
- openaiImageInstance = new OpenAIImageProvider();
351
- await openaiImageInstance.initialize({ apiKey: imageApiKey });
352
- } else if (imageProvider === "gemini") {
353
- geminiInstance = new GeminiProvider();
354
- await geminiInstance.initialize({ apiKey: imageApiKey });
355
- } else if (imageProvider === "grok") {
356
- grokInstance = new GrokProvider();
357
- await grokInstance.initialize({ apiKey: imageApiKey });
358
- }
359
-
360
- // Get character description from first segment (should be same across all)
361
- const characterDescription = segments[0]?.characterDescription;
362
-
363
- for (let i = 0; i < segments.length; i++) {
364
- const segment = segments[i];
365
- imageSpinner.text = `🎨 Generating image ${i + 1}/${segments.length}: ${segment.description.slice(0, 30)}...`;
366
-
367
- // Build comprehensive image prompt with character description
368
- let imagePrompt = segment.visuals;
369
-
370
- // Add character description to ensure consistency
371
- if (characterDescription) {
372
- imagePrompt = `CHARACTER (must match exactly): ${characterDescription}. SCENE: ${imagePrompt}`;
373
- }
374
-
375
- // Add visual style
376
- if (segment.visualStyle) {
377
- imagePrompt = `${imagePrompt}. STYLE: ${segment.visualStyle}`;
378
- }
379
-
380
- // For scenes after the first, add extra continuity instruction (OpenAI)
381
- // Gemini uses editImage with reference instead
382
- if (i > 0 && firstSceneImage && imageProvider !== "gemini") {
383
- imagePrompt = `${imagePrompt}. CRITICAL: The character must look IDENTICAL to the first scene - same face, hair, clothing, accessories.`;
384
- }
385
-
386
- try {
387
- let imageBuffer: Buffer | undefined;
388
- let imageUrl: string | undefined;
389
- let imageError: string | undefined;
390
-
391
- if ((imageProvider === "openai" || imageProvider === "dalle") && openaiImageInstance) {
392
- const imageResult = await openaiImageInstance.generateImage(imagePrompt, {
393
- size: dalleImageSizes[options.aspectRatio] || "1536x1024",
394
- quality: "standard",
395
- });
396
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
397
- // GPT Image 1.5 returns base64, DALL-E 3 returns URL
398
- const img = imageResult.images[0];
399
- if (img.base64) {
400
- imageBuffer = Buffer.from(img.base64, "base64");
401
- } else if (img.url) {
402
- imageUrl = img.url;
403
- }
404
- } else {
405
- imageError = imageResult.error;
406
- }
407
- } else if (imageProvider === "gemini" && geminiInstance) {
408
- // Gemini: use editImage with first scene reference for subsequent scenes
409
- if (i > 0 && firstSceneImage) {
410
- // Use editImage to maintain style continuity with first scene
411
- const editPrompt = `Create a new scene for a video: ${imagePrompt}. IMPORTANT: Maintain the exact same character appearance, clothing, environment style, color palette, and art style as the reference image.`;
412
- const imageResult = await geminiInstance.editImage([firstSceneImage], editPrompt, {
413
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
414
- });
415
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
416
- const img = imageResult.images[0];
417
- if (img.base64) {
418
- imageBuffer = Buffer.from(img.base64, "base64");
419
- }
420
- } else {
421
- imageError = imageResult.error;
422
- }
423
- } else {
424
- // First scene: use regular generateImage
425
- const imageResult = await geminiInstance.generateImage(imagePrompt, {
426
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
427
- });
428
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
429
- const img = imageResult.images[0];
430
- if (img.base64) {
431
- imageBuffer = Buffer.from(img.base64, "base64");
432
- }
433
- } else {
434
- imageError = imageResult.error;
435
- }
436
- }
437
- } else if (imageProvider === "grok" && grokInstance) {
438
- const imageResult = await grokInstance.generateImage(imagePrompt, {
439
- aspectRatio: options.aspectRatio || "16:9",
440
- });
441
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
442
- const img = imageResult.images[0];
443
- if (img.base64) {
444
- imageBuffer = Buffer.from(img.base64, "base64");
445
- } else if (img.url) {
446
- imageUrl = img.url;
447
- }
448
- } else {
449
- imageError = imageResult.error;
450
- }
451
- }
452
-
453
- // Save the image
454
- const imagePath = resolve(outputDir, `scene-${i + 1}.png`);
455
-
456
- if (imageBuffer) {
457
- await writeFile(imagePath, imageBuffer);
458
- imagePaths.push(imagePath);
459
- // Store first successful image for style continuity
460
- if (!firstSceneImage) {
461
- firstSceneImage = imageBuffer;
462
- }
463
- } else if (imageUrl) {
464
- const response = await fetch(imageUrl);
465
- const buffer = Buffer.from(await response.arrayBuffer());
466
- await writeFile(imagePath, buffer);
467
- imagePaths.push(imagePath);
468
- // Store first successful image for style continuity
469
- if (!firstSceneImage) {
470
- firstSceneImage = buffer;
471
- }
472
- } else {
473
- const errorMsg = imageError || "Unknown error";
474
- console.log(chalk.yellow(`\n ⚠ Failed to generate image for scene ${i + 1}: ${errorMsg}`));
475
- imagePaths.push("");
476
- }
477
- } catch (err) {
478
- console.log(chalk.yellow(`\n ⚠ Error generating image for scene ${i + 1}: ${err}`));
479
- imagePaths.push("");
480
- }
481
-
482
- // Small delay to avoid rate limiting
483
- if (i < segments.length - 1) {
484
- await new Promise((r) => setTimeout(r, 500));
485
- }
486
- }
487
-
488
- const successfulImages = imagePaths.filter((p) => p !== "").length;
489
- imageSpinner.succeed(chalk.green(`Generated ${successfulImages}/${segments.length} images with ${providerNames[imageProvider]}`));
490
- console.log();
491
-
492
- // Step 4: Generate videos (if not images-only)
493
- const videoPaths: string[] = [];
494
- const failedScenes: number[] = []; // Track failed scenes for summary
495
- const maxRetries = parseInt(options.retries) || DEFAULT_VIDEO_RETRIES;
496
-
497
- if (!options.imagesOnly && videoApiKey) {
498
- const videoSpinner = ora(`🎬 Generating videos with ${options.generator === "kling" ? "Kling" : "Runway"}...`).start();
499
-
500
- if (options.generator === "kling") {
501
- const kling = new KlingProvider();
502
- await kling.initialize({ apiKey: videoApiKey });
503
-
504
- if (!kling.isConfigured()) {
505
- videoSpinner.fail(chalk.red("Invalid Kling API key format. Use ACCESS_KEY:SECRET_KEY"));
506
- process.exit(1);
507
- }
508
-
509
- // Check for ImgBB API key for image-to-video support (from config or env)
510
- const imgbbApiKey = await getApiKeyFromConfig("imgbb") || process.env.IMGBB_API_KEY;
511
- const useImageToVideo = !!imgbbApiKey;
512
-
513
- if (useImageToVideo) {
514
- videoSpinner.text = `🎬 Uploading images to ImgBB for image-to-video...`;
515
- }
516
-
517
- // Upload images to ImgBB if API key is available (for Kling v2.x image-to-video)
518
- const imageUrls: (string | undefined)[] = [];
519
- if (useImageToVideo) {
520
- for (let i = 0; i < imagePaths.length; i++) {
521
- if (imagePaths[i] && imagePaths[i] !== "") {
522
- try {
523
- const imageBuffer = await readFile(imagePaths[i]);
524
- const uploadResult = await uploadToImgbb(imageBuffer, imgbbApiKey);
525
- if (uploadResult.success && uploadResult.url) {
526
- imageUrls[i] = uploadResult.url;
527
- } else {
528
- console.log(chalk.yellow(`\n ⚠ Failed to upload image ${i + 1}: ${uploadResult.error}`));
529
- imageUrls[i] = undefined;
530
- }
531
- } catch {
532
- imageUrls[i] = undefined;
533
- }
534
- } else {
535
- imageUrls[i] = undefined;
536
- }
537
- }
538
- const uploadedCount = imageUrls.filter((u) => u).length;
539
- if (uploadedCount > 0) {
540
- videoSpinner.text = `🎬 Uploaded ${uploadedCount}/${imagePaths.length} images to ImgBB`;
541
- }
542
- }
543
-
544
- // Sequential mode: generate one video at a time (slower but more reliable)
545
- if (options.sequential) {
546
- for (let i = 0; i < segments.length; i++) {
547
- const segment = segments[i] as StoryboardSegment;
548
- videoSpinner.text = `🎬 Scene ${i + 1}/${segments.length}: Starting...`;
549
-
550
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
551
- const referenceImage = imageUrls[i];
552
-
553
- let completed = false;
554
- for (let attempt = 0; attempt <= maxRetries && !completed; attempt++) {
555
- const result = await generateVideoWithRetryKling(
556
- kling,
557
- segment,
558
- {
559
- duration: videoDuration,
560
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
561
- referenceImage,
562
- },
563
- 0, // Handle retries at this level
564
- (msg) => {
565
- videoSpinner.text = `🎬 Scene ${i + 1}/${segments.length}: ${msg}`;
566
- }
567
- );
568
-
569
- if (!result) {
570
- if (attempt < maxRetries) {
571
- videoSpinner.text = `🎬 Scene ${i + 1}: Submit failed, retry ${attempt + 1}/${maxRetries}...`;
572
- await sleep(RETRY_DELAY_MS);
573
- continue;
574
- }
575
- console.log(chalk.yellow(`\n ⚠ Failed to start video generation for scene ${i + 1}`));
576
- videoPaths[i] = "";
577
- failedScenes.push(i + 1);
578
- break;
579
- }
580
-
581
- try {
582
- const waitResult = await kling.waitForCompletion(
583
- result.taskId,
584
- result.type,
585
- (status) => {
586
- videoSpinner.text = `🎬 Scene ${i + 1}/${segments.length}: ${status.status}...`;
587
- },
588
- 600000
589
- );
590
-
591
- if (waitResult.status === "completed" && waitResult.videoUrl) {
592
- const videoPath = resolve(outputDir, `scene-${i + 1}.mp4`);
593
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
594
- await writeFile(videoPath, buffer);
595
-
596
- // Extend video to match narration duration if needed
597
- await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${i + 1}`, {
598
- kling,
599
- videoId: waitResult.videoId,
600
- onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
601
- });
602
-
603
- videoPaths[i] = videoPath;
604
- completed = true;
605
- console.log(chalk.green(`\n ✓ Scene ${i + 1} completed`));
606
- } else if (attempt < maxRetries) {
607
- videoSpinner.text = `🎬 Scene ${i + 1}: Failed, retry ${attempt + 1}/${maxRetries}...`;
608
- await sleep(RETRY_DELAY_MS);
609
- } else {
610
- videoPaths[i] = "";
611
- failedScenes.push(i + 1);
612
- }
613
- } catch (err) {
614
- if (attempt < maxRetries) {
615
- videoSpinner.text = `🎬 Scene ${i + 1}: Error, retry ${attempt + 1}/${maxRetries}...`;
616
- await sleep(RETRY_DELAY_MS);
617
- } else {
618
- console.log(chalk.yellow(`\n ⚠ Error for scene ${i + 1}: ${err}`));
619
- videoPaths[i] = "";
620
- failedScenes.push(i + 1);
621
- }
622
- }
623
- }
624
- }
625
- } else {
626
- // Parallel mode (default): batch-based submission respecting concurrency limit
627
- const concurrency = Math.max(1, parseInt(options.concurrency) || 3);
628
-
629
- for (let batchStart = 0; batchStart < segments.length; batchStart += concurrency) {
630
- const batchEnd = Math.min(batchStart + concurrency, segments.length);
631
- const batchNum = Math.floor(batchStart / concurrency) + 1;
632
- const totalBatches = Math.ceil(segments.length / concurrency);
633
-
634
- if (totalBatches > 1) {
635
- videoSpinner.text = `🎬 Batch ${batchNum}/${totalBatches}: submitting scenes ${batchStart + 1}-${batchEnd}...`;
636
- }
637
-
638
- // Phase 1: Submit batch
639
- const tasks: Array<{ taskId: string; index: number; segment: StoryboardSegment; type: "text2video" | "image2video" }> = [];
640
-
641
- for (let i = batchStart; i < batchEnd; i++) {
642
- const segment = segments[i] as StoryboardSegment;
643
- videoSpinner.text = `🎬 Submitting video task ${i + 1}/${segments.length}...`;
644
-
645
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
646
- const referenceImage = imageUrls[i];
647
-
648
- const result = await generateVideoWithRetryKling(
649
- kling,
650
- segment,
651
- {
652
- duration: videoDuration,
653
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
654
- referenceImage,
655
- },
656
- maxRetries,
657
- (msg) => {
658
- videoSpinner.text = `🎬 Scene ${i + 1}: ${msg}`;
659
- }
660
- );
661
-
662
- if (result) {
663
- tasks.push({ taskId: result.taskId, index: i, segment, type: result.type });
664
- if (!videoPaths[i]) videoPaths[i] = "";
665
- } else {
666
- console.log(chalk.yellow(`\n ⚠ Failed to start video generation for scene ${i + 1} (after ${maxRetries} retries)`));
667
- videoPaths[i] = "";
668
- failedScenes.push(i + 1);
669
- }
670
- }
671
-
672
- // Phase 2: Wait for batch completion
673
- videoSpinner.text = `🎬 Waiting for batch ${batchNum} (${tasks.length} video${tasks.length > 1 ? "s" : ""})...`;
674
-
675
- for (const task of tasks) {
676
- let completed = false;
677
- let currentTaskId = task.taskId;
678
- let currentType = task.type;
679
-
680
- for (let attempt = 0; attempt <= maxRetries && !completed; attempt++) {
681
- try {
682
- const result = await kling.waitForCompletion(
683
- currentTaskId,
684
- currentType,
685
- (status) => {
686
- videoSpinner.text = `🎬 Scene ${task.index + 1}: ${status.status}...`;
687
- },
688
- 600000
689
- );
690
-
691
- if (result.status === "completed" && result.videoUrl) {
692
- const videoPath = resolve(outputDir, `scene-${task.index + 1}.mp4`);
693
- const buffer = await downloadVideo(result.videoUrl, videoApiKey);
694
- await writeFile(videoPath, buffer);
695
-
696
- // Extend video to match narration duration if needed
697
- await extendVideoToTarget(videoPath, task.segment.duration, outputDir, `Scene ${task.index + 1}`, {
698
- kling,
699
- videoId: result.videoId,
700
- onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
701
- });
702
-
703
- videoPaths[task.index] = videoPath;
704
- completed = true;
705
- } else if (attempt < maxRetries) {
706
- videoSpinner.text = `🎬 Scene ${task.index + 1}: Retry ${attempt + 1}/${maxRetries}...`;
707
- await sleep(RETRY_DELAY_MS);
708
-
709
- const videoDuration = (task.segment.duration > 5 ? 10 : 5) as 5 | 10;
710
- const retryReferenceImage = imageUrls[task.index];
711
-
712
- const retryResult = await generateVideoWithRetryKling(
713
- kling,
714
- task.segment,
715
- {
716
- duration: videoDuration,
717
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
718
- referenceImage: retryReferenceImage,
719
- },
720
- 0
721
- );
722
-
723
- if (retryResult) {
724
- currentTaskId = retryResult.taskId;
725
- currentType = retryResult.type;
726
- } else {
727
- videoPaths[task.index] = "";
728
- failedScenes.push(task.index + 1);
729
- completed = true;
730
- }
731
- } else {
732
- videoPaths[task.index] = "";
733
- failedScenes.push(task.index + 1);
734
- }
735
- } catch (err) {
736
- if (attempt >= maxRetries) {
737
- console.log(chalk.yellow(`\n ⚠ Error completing video for scene ${task.index + 1}: ${err}`));
738
- videoPaths[task.index] = "";
739
- failedScenes.push(task.index + 1);
740
- } else {
741
- videoSpinner.text = `🎬 Scene ${task.index + 1}: Error, retry ${attempt + 1}/${maxRetries}...`;
742
- await sleep(RETRY_DELAY_MS);
743
- }
744
- }
745
- }
746
- }
747
-
748
- if (totalBatches > 1 && batchEnd < segments.length) {
749
- console.log(chalk.dim(` → Batch ${batchNum}/${totalBatches} complete`));
750
- }
751
- }
752
- }
753
- } else {
754
- // Runway
755
- const runway = new RunwayProvider();
756
- await runway.initialize({ apiKey: videoApiKey });
757
-
758
- // Submit all video generation tasks with retry logic
759
- const tasks: Array<{ taskId: string; index: number; imagePath: string; referenceImage: string; segment: StoryboardSegment }> = [];
760
-
761
- for (let i = 0; i < segments.length; i++) {
762
- if (!imagePaths[i]) {
763
- videoPaths.push("");
764
- continue;
765
- }
766
-
767
- const segment = segments[i] as StoryboardSegment;
768
- videoSpinner.text = `🎬 Submitting video task ${i + 1}/${segments.length}...`;
769
-
770
- const imageBuffer = await readFile(imagePaths[i]);
771
- const ext = extname(imagePaths[i]).toLowerCase().slice(1);
772
- const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
773
- const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
774
-
775
- // Use 10s video if narration > 5s to avoid video ending before narration
776
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
777
-
778
- const result = await generateVideoWithRetryRunway(
779
- runway,
780
- segment,
781
- referenceImage,
782
- {
783
- duration: videoDuration,
784
- aspectRatio: options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio as "16:9" | "9:16"),
785
- },
786
- maxRetries,
787
- (msg) => {
788
- videoSpinner.text = `🎬 Scene ${i + 1}: ${msg}`;
789
- }
790
- );
791
-
792
- if (result) {
793
- tasks.push({ taskId: result.taskId, index: i, imagePath: imagePaths[i], referenceImage, segment });
794
- } else {
795
- console.log(chalk.yellow(`\n ⚠ Failed to start video generation for scene ${i + 1} (after ${maxRetries} retries)`));
796
- videoPaths[i] = "";
797
- failedScenes.push(i + 1);
798
- }
799
- }
800
-
801
- // Wait for all tasks to complete with retry logic
802
- videoSpinner.text = `🎬 Waiting for ${tasks.length} video(s) to complete...`;
803
-
804
- for (const task of tasks) {
805
- let completed = false;
806
- let currentTaskId = task.taskId;
807
-
808
- for (let attempt = 0; attempt <= maxRetries && !completed; attempt++) {
809
- try {
810
- const result = await runway.waitForCompletion(
811
- currentTaskId,
812
- (status) => {
813
- const progress = status.progress !== undefined ? `${status.progress}%` : status.status;
814
- videoSpinner.text = `🎬 Scene ${task.index + 1}: ${progress}...`;
815
- },
816
- 300000 // 5 minute timeout per video
817
- );
818
-
819
- if (result.status === "completed" && result.videoUrl) {
820
- const videoPath = resolve(outputDir, `scene-${task.index + 1}.mp4`);
821
- const buffer = await downloadVideo(result.videoUrl, videoApiKey);
822
- await writeFile(videoPath, buffer);
823
-
824
- // Extend video to match narration duration if needed
825
- await extendVideoToTarget(videoPath, task.segment.duration, outputDir, `Scene ${task.index + 1}`, {
826
- onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
827
- });
828
-
829
- videoPaths[task.index] = videoPath;
830
- completed = true;
831
- } else if (attempt < maxRetries) {
832
- // Resubmit task on failure
833
- videoSpinner.text = `🎬 Scene ${task.index + 1}: Retry ${attempt + 1}/${maxRetries}...`;
834
- await sleep(RETRY_DELAY_MS);
835
-
836
- const videoDuration = (task.segment.duration > 5 ? 10 : 5) as 5 | 10;
837
- const retryResult = await generateVideoWithRetryRunway(
838
- runway,
839
- task.segment,
840
- task.referenceImage,
841
- {
842
- duration: videoDuration,
843
- aspectRatio: options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio as "16:9" | "9:16"),
844
- },
845
- 0, // No nested retries
846
- (msg) => {
847
- videoSpinner.text = `🎬 Scene ${task.index + 1}: ${msg}`;
848
- }
849
- );
850
-
851
- if (retryResult) {
852
- currentTaskId = retryResult.taskId;
853
- } else {
854
- videoPaths[task.index] = "";
855
- failedScenes.push(task.index + 1);
856
- completed = true; // Exit retry loop
857
- }
858
- } else {
859
- videoPaths[task.index] = "";
860
- failedScenes.push(task.index + 1);
861
- }
862
- } catch (err) {
863
- if (attempt >= maxRetries) {
864
- console.log(chalk.yellow(`\n ⚠ Error completing video for scene ${task.index + 1}: ${err}`));
865
- videoPaths[task.index] = "";
866
- failedScenes.push(task.index + 1);
867
- } else {
868
- videoSpinner.text = `🎬 Scene ${task.index + 1}: Error, retry ${attempt + 1}/${maxRetries}...`;
869
- await sleep(RETRY_DELAY_MS);
870
- }
871
- }
872
- }
873
- }
874
- }
875
-
876
- const successfulVideos = videoPaths.filter((p) => p && p !== "").length;
877
- videoSpinner.succeed(chalk.green(`Generated ${successfulVideos}/${segments.length} videos`));
878
- console.log();
879
- }
880
-
881
- // Step 4.5: Apply text overlays (if segments have textOverlays)
882
- if (options.textOverlay !== false) {
883
- const overlaySegments = segments.filter(
884
- (s: StoryboardSegment, i: number) => s.textOverlays && s.textOverlays.length > 0 && videoPaths[i] && videoPaths[i] !== ""
885
- );
886
- if (overlaySegments.length > 0) {
887
- const overlaySpinner = ora(`Applying text overlays to ${overlaySegments.length} scene(s)...`).start();
888
- let overlayCount = 0;
889
- for (let i = 0; i < segments.length; i++) {
890
- const segment = segments[i] as StoryboardSegment;
891
- if (segment.textOverlays && segment.textOverlays.length > 0 && videoPaths[i] && videoPaths[i] !== "") {
892
- try {
893
- const overlayOutput = videoPaths[i].replace(/(\.[^.]+)$/, "-overlay$1");
894
- const overlayResult = await applyTextOverlays({
895
- videoPath: videoPaths[i],
896
- texts: segment.textOverlays,
897
- outputPath: overlayOutput,
898
- style: (options.textStyle as TextOverlayStyle) || "lower-third",
899
- });
900
- if (overlayResult.success && overlayResult.outputPath) {
901
- videoPaths[i] = overlayResult.outputPath;
902
- overlayCount++;
903
- }
904
- } catch {
905
- // Silent fallback: keep original video
906
- }
907
- }
908
- }
909
- overlaySpinner.succeed(chalk.green(`Applied text overlays to ${overlayCount} scene(s)`));
910
- console.log();
911
- }
912
- }
913
-
914
- // Step 5: Assemble project
915
- const assembleSpinner = ora("Assembling project...").start();
916
-
917
- const project = new Project("Script-to-Video Output");
918
- project.setAspectRatio(options.aspectRatio as "16:9" | "9:16" | "1:1");
919
-
920
- // Clear default tracks and create new ones
921
- const defaultTracks = project.getTracks();
922
- for (const track of defaultTracks) {
923
- project.removeTrack(track.id);
924
- }
925
-
926
- const videoTrack = project.addTrack({
927
- name: "Video",
928
- type: "video",
929
- order: 1,
930
- isMuted: false,
931
- isLocked: false,
932
- isVisible: true,
933
- });
934
-
935
- const audioTrack = project.addTrack({
936
- name: "Audio",
937
- type: "audio",
938
- order: 0,
939
- isMuted: false,
940
- isLocked: false,
941
- isVisible: true,
942
- });
943
-
944
- // Add per-scene narration sources and clips
945
- for (const tts of perSceneTTS) {
946
- const segment = segments[tts.segmentIndex];
947
- const audioSource = project.addSource({
948
- name: `Narration ${tts.segmentIndex + 1}`,
949
- url: tts.path,
950
- type: "audio",
951
- duration: tts.duration,
952
- });
953
-
954
- project.addClip({
955
- sourceId: audioSource.id,
956
- trackId: audioTrack.id,
957
- startTime: segment.startTime,
958
- duration: tts.duration,
959
- sourceStartOffset: 0,
960
- sourceEndOffset: tts.duration,
961
- });
962
- }
963
-
964
- // Add video/image sources and clips
965
- let currentTime = 0;
966
- const videoClipIds: string[] = [];
967
- const fadeDuration = 0.3; // Fade duration in seconds for smooth transitions
968
-
969
- for (let i = 0; i < segments.length; i++) {
970
- const segment = segments[i];
971
- const hasVideo = videoPaths[i] && videoPaths[i] !== "";
972
- const hasImage = imagePaths[i] && imagePaths[i] !== "";
973
-
974
- if (!hasVideo && !hasImage) {
975
- // Skip if no visual asset
976
- currentTime += segment.duration;
977
- continue;
978
- }
979
-
980
- const assetPath = hasVideo ? videoPaths[i] : imagePaths[i];
981
- const mediaType = hasVideo ? "video" : "image";
982
-
983
- const source = project.addSource({
984
- name: `Scene ${i + 1}`,
985
- url: assetPath,
986
- type: mediaType as "video" | "image",
987
- duration: segment.duration,
988
- });
989
-
990
- const clip = project.addClip({
991
- sourceId: source.id,
992
- trackId: videoTrack.id,
993
- startTime: currentTime,
994
- duration: segment.duration,
995
- sourceStartOffset: 0,
996
- sourceEndOffset: segment.duration,
997
- });
998
-
999
- videoClipIds.push(clip.id);
1000
- currentTime += segment.duration;
1001
- }
1002
-
1003
- // Add fade effects to video clips for smoother scene transitions
1004
- for (let i = 0; i < videoClipIds.length; i++) {
1005
- const clipId = videoClipIds[i];
1006
- const clip = project.getClips().find(c => c.id === clipId);
1007
- if (!clip) continue;
1008
-
1009
- // Add fadeIn effect (except for first clip)
1010
- if (i > 0) {
1011
- project.addEffect(clipId, {
1012
- type: "fadeIn",
1013
- startTime: 0,
1014
- duration: fadeDuration,
1015
- params: {},
1016
- });
1017
- }
1018
-
1019
- // Add fadeOut effect (except for last clip)
1020
- if (i < videoClipIds.length - 1) {
1021
- project.addEffect(clipId, {
1022
- type: "fadeOut",
1023
- startTime: clip.duration - fadeDuration,
1024
- duration: fadeDuration,
1025
- params: {},
1026
- });
1027
- }
1028
- }
1029
-
1030
- // Save project file
1031
- let outputPath = resolve(process.cwd(), options.output);
1032
-
1033
- // Detect if output looks like a directory (ends with / or no .json extension)
1034
- const looksLikeDirectory =
1035
- options.output.endsWith("/") ||
1036
- (!options.output.endsWith(".json") &&
1037
- !options.output.endsWith(".vibe.json"));
1038
-
1039
- if (looksLikeDirectory) {
1040
- // Create directory if it doesn't exist
1041
- if (!existsSync(outputPath)) {
1042
- await mkdir(outputPath, { recursive: true });
1043
- }
1044
- outputPath = resolve(outputPath, "project.vibe.json");
1045
- } else if (
1046
- existsSync(outputPath) &&
1047
- (await stat(outputPath)).isDirectory()
1048
- ) {
1049
- // Existing directory without trailing slash
1050
- outputPath = resolve(outputPath, "project.vibe.json");
1051
- } else {
1052
- // File path - ensure parent directory exists
1053
- const parentDir = dirname(outputPath);
1054
- if (!existsSync(parentDir)) {
1055
- await mkdir(parentDir, { recursive: true });
1056
- }
1057
- }
1058
-
1059
- await writeFile(
1060
- outputPath,
1061
- JSON.stringify(project.toJSON(), null, 2),
1062
- "utf-8"
1063
- );
1064
-
1065
- assembleSpinner.succeed(chalk.green("Project assembled"));
1066
-
1067
- // Step 6: AI Review (optional)
1068
- if (options.review) {
1069
- const reviewSpinner = ora("Reviewing video with Gemini AI...").start();
1070
- try {
1071
- const reviewTarget = videoPaths.find((p) => p && p !== "");
1072
- if (reviewTarget) {
1073
- const storyboardFile = resolve(effectiveOutputDir, "storyboard.json");
1074
- const reviewResult = await executeReview({
1075
- videoPath: reviewTarget,
1076
- storyboardPath: existsSync(storyboardFile) ? storyboardFile : undefined,
1077
- autoApply: options.reviewAutoApply,
1078
- model: "flash",
1079
- });
1080
-
1081
- if (reviewResult.success && reviewResult.feedback) {
1082
- reviewSpinner.succeed(chalk.green(`AI Review: ${reviewResult.feedback.overallScore}/10`));
1083
- if (reviewResult.appliedFixes && reviewResult.appliedFixes.length > 0) {
1084
- for (const fix of reviewResult.appliedFixes) {
1085
- console.log(chalk.green(` + ${fix}`));
1086
- }
1087
- }
1088
- if (reviewResult.feedback.recommendations.length > 0) {
1089
- for (const rec of reviewResult.feedback.recommendations) {
1090
- console.log(chalk.dim(` * ${rec}`));
1091
- }
1092
- }
1093
- } else {
1094
- reviewSpinner.warn(chalk.yellow("AI review completed but no actionable feedback"));
1095
- }
1096
- } else {
1097
- reviewSpinner.warn(chalk.yellow("No videos available for review"));
1098
- }
1099
- } catch {
1100
- reviewSpinner.warn(chalk.yellow("AI review skipped (non-critical error)"));
1101
- }
1102
- console.log();
1103
- }
1104
-
1105
- // Final summary
1106
- console.log();
1107
- console.log(chalk.bold.green("Script-to-Video complete!"));
1108
- console.log(chalk.dim("─".repeat(60)));
1109
- console.log();
1110
- console.log(` 📄 Project: ${chalk.cyan(outputPath)}`);
1111
- console.log(` 🎬 Scenes: ${segments.length}`);
1112
- console.log(` ⏱️ Duration: ${totalDuration}s`);
1113
- console.log(` 📁 Assets: ${effectiveOutputDir}/`);
1114
- if (perSceneTTS.length > 0 || failedNarrations.length > 0) {
1115
- const narrationInfo = `${perSceneTTS.length}/${segments.length}`;
1116
- if (failedNarrations.length > 0) {
1117
- const failedSceneNums = failedNarrations.map((f) => f.sceneNum).join(", ");
1118
- console.log(` 🎙️ Narrations: ${narrationInfo} narration-*.mp3`);
1119
- console.log(chalk.yellow(` ⚠ Failed: scene ${failedSceneNums}`));
1120
- } else {
1121
- console.log(` 🎙️ Narrations: ${perSceneTTS.length} narration-*.mp3`);
1122
- }
1123
- }
1124
- console.log(` 🖼️ Images: ${successfulImages} scene-*.png`);
1125
- if (!options.imagesOnly) {
1126
- const videoCount = videoPaths.filter((p) => p && p !== "").length;
1127
- console.log(` 🎥 Videos: ${videoCount}/${segments.length} scene-*.mp4`);
1128
- if (failedScenes.length > 0) {
1129
- const uniqueFailedScenes = [...new Set(failedScenes)].sort((a, b) => a - b);
1130
- console.log(chalk.yellow(` ⚠ Failed: scene ${uniqueFailedScenes.join(", ")} (fallback to image)`));
1131
- }
1132
- }
1133
- console.log();
1134
- console.log(chalk.dim("Next steps:"));
1135
- console.log(chalk.dim(` vibe project info ${options.output}`));
1136
- console.log(chalk.dim(` vibe export ${options.output} -o final.mp4`));
1137
-
1138
- // Show regeneration hint if there were failures
1139
- if (!options.imagesOnly && failedScenes.length > 0) {
1140
- const uniqueFailedScenes = [...new Set(failedScenes)].sort((a, b) => a - b);
1141
- console.log();
1142
- console.log(chalk.dim("💡 To regenerate failed scenes:"));
1143
- for (const sceneNum of uniqueFailedScenes) {
1144
- console.log(chalk.dim(` vibe ai regenerate-scene ${effectiveOutputDir}/ --scene ${sceneNum} --video-only`));
1145
- }
1146
- }
1147
- console.log();
1148
- } catch (error) {
1149
- console.error(chalk.red("Script-to-Video failed"));
1150
- console.error(error);
1151
- process.exit(1);
1152
- }
1153
- });
1154
-
1155
- // Regenerate Scene command
1156
- aiCommand
1157
- .command("regenerate-scene")
1158
- .description("Regenerate a specific scene in a script-to-video project")
1159
- .argument("<project-dir>", "Path to the script-to-video output directory")
1160
- .requiredOption("--scene <numbers>", "Scene number(s) to regenerate (1-based), e.g., 3 or 3,4,5")
1161
- .option("--video-only", "Only regenerate video")
1162
- .option("--narration-only", "Only regenerate narration")
1163
- .option("--image-only", "Only regenerate image")
1164
- .option("-g, --generator <engine>", "Video generator: kling | runway | veo", "kling")
1165
- .option("-i, --image-provider <provider>", "Image provider: gemini | openai | grok", "gemini")
1166
- .option("-v, --voice <id>", "ElevenLabs voice ID for narration")
1167
- .option("-a, --aspect-ratio <ratio>", "Aspect ratio: 16:9 | 9:16 | 1:1", "16:9")
1168
- .option("--retries <count>", "Number of retries for video generation failures", String(DEFAULT_VIDEO_RETRIES))
1169
- .option("--reference-scene <num>", "Use another scene's image as reference for character consistency")
1170
- .action(async (projectDir: string, options) => {
1171
- try {
1172
- const outputDir = resolve(process.cwd(), projectDir);
1173
- const storyboardPath = resolve(outputDir, "storyboard.json");
1174
- const projectPath = resolve(outputDir, "project.vibe.json");
1175
-
1176
- // Validate project directory
1177
- if (!existsSync(outputDir)) {
1178
- console.error(chalk.red(`Project directory not found: ${outputDir}`));
1179
- process.exit(1);
1180
- }
1181
-
1182
- if (!existsSync(storyboardPath)) {
1183
- console.error(chalk.red(`Storyboard not found: ${storyboardPath}`));
1184
- console.error(chalk.dim("This command requires a storyboard.json file from script-to-video output"));
1185
- process.exit(1);
1186
- }
1187
-
1188
- // Parse scene number(s) - supports "3" or "3,4,5"
1189
- const sceneNums = options.scene.split(",").map((s: string) => parseInt(s.trim())).filter((n: number) => !isNaN(n) && n >= 1);
1190
- if (sceneNums.length === 0) {
1191
- console.error(chalk.red("Scene number must be a positive integer (1-based), e.g., --scene 3 or --scene 3,4,5"));
1192
- process.exit(1);
1193
- }
1194
-
1195
- // Load storyboard
1196
- const storyboardContent = await readFile(storyboardPath, "utf-8");
1197
- const segments: StoryboardSegment[] = JSON.parse(storyboardContent);
1198
-
1199
- // Validate all scene numbers
1200
- for (const sceneNum of sceneNums) {
1201
- if (sceneNum > segments.length) {
1202
- console.error(chalk.red(`Scene ${sceneNum} does not exist. Storyboard has ${segments.length} scenes.`));
1203
- process.exit(1);
1204
- }
1205
- }
1206
-
1207
- // Determine what to regenerate
1208
- const regenerateVideo = options.videoOnly || (!options.narrationOnly && !options.imageOnly);
1209
- const regenerateNarration = options.narrationOnly || (!options.videoOnly && !options.imageOnly);
1210
- const regenerateImage = options.imageOnly || (!options.videoOnly && !options.narrationOnly);
1211
-
1212
- console.log();
1213
- console.log(chalk.bold.cyan(`🔄 Regenerating Scene${sceneNums.length > 1 ? "s" : ""} ${sceneNums.join(", ")}`));
1214
- console.log(chalk.dim("─".repeat(60)));
1215
- console.log();
1216
- console.log(` 📁 Project: ${outputDir}`);
1217
- console.log(` 🎬 Scenes: ${sceneNums.join(", ")} of ${segments.length}`);
1218
- console.log();
1219
-
1220
- // Get required API keys (once, before processing scenes)
1221
- let imageApiKey: string | undefined;
1222
- let videoApiKey: string | undefined;
1223
- let elevenlabsApiKey: string | undefined;
1224
-
1225
- if (regenerateImage) {
1226
- const imageProvider = options.imageProvider || "openai";
1227
- if (imageProvider === "openai" || imageProvider === "dalle") {
1228
- imageApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
1229
- if (!imageApiKey) {
1230
- console.error(chalk.red("OpenAI API key required for image generation. Set OPENAI_API_KEY in .env or run: vibe setup"));
1231
- process.exit(1);
1232
- }
1233
- } else if (imageProvider === "gemini") {
1234
- imageApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
1235
- if (!imageApiKey) {
1236
- console.error(chalk.red("Google API key required for Gemini image generation. Set GOOGLE_API_KEY in .env or run: vibe setup"));
1237
- process.exit(1);
1238
- }
1239
- } else if (imageProvider === "grok") {
1240
- imageApiKey = (await getApiKey("XAI_API_KEY", "xAI")) ?? undefined;
1241
- if (!imageApiKey) {
1242
- console.error(chalk.red("xAI API key required for Grok image generation. Set XAI_API_KEY in .env or run: vibe setup"));
1243
- process.exit(1);
1244
- }
1245
- }
1246
- }
1247
-
1248
- if (regenerateVideo) {
1249
- if (options.generator === "kling") {
1250
- const key = await getApiKey("KLING_API_KEY", "Kling");
1251
- if (!key) {
1252
- console.error(chalk.red("Kling API key required. Set KLING_API_KEY in .env or run: vibe setup"));
1253
- process.exit(1);
1254
- }
1255
- videoApiKey = key;
1256
- } else {
1257
- const key = await getApiKey("RUNWAY_API_SECRET", "Runway");
1258
- if (!key) {
1259
- console.error(chalk.red("Runway API key required. Set RUNWAY_API_SECRET in .env or run: vibe setup"));
1260
- process.exit(1);
1261
- }
1262
- videoApiKey = key;
1263
- }
1264
- }
1265
-
1266
- if (regenerateNarration) {
1267
- const key = await getApiKey("ELEVENLABS_API_KEY", "ElevenLabs");
1268
- if (!key) {
1269
- console.error(chalk.red("ElevenLabs API key required for narration. Set ELEVENLABS_API_KEY in .env or run: vibe setup"));
1270
- process.exit(1);
1271
- }
1272
- elevenlabsApiKey = key;
1273
- }
1274
-
1275
- // Process each scene
1276
- for (const sceneNum of sceneNums) {
1277
- const segment = segments[sceneNum - 1];
1278
-
1279
- console.log(chalk.cyan(`\n── Scene ${sceneNum} ──`));
1280
- console.log(chalk.dim(` ${segment.description.slice(0, 50)}...`));
1281
-
1282
- // Step 1: Regenerate narration if needed
1283
- const narrationPath = resolve(outputDir, `narration-${sceneNum}.mp3`);
1284
- let narrationDuration = segment.duration;
1285
-
1286
- if (regenerateNarration && elevenlabsApiKey) {
1287
- const ttsSpinner = ora(`🎙️ Regenerating narration for scene ${sceneNum}...`).start();
1288
-
1289
- const elevenlabs = new ElevenLabsProvider();
1290
- await elevenlabs.initialize({ apiKey: elevenlabsApiKey });
1291
-
1292
- const narrationText = segment.narration || segment.description;
1293
-
1294
- const ttsResult = await elevenlabs.textToSpeech(narrationText, {
1295
- voiceId: options.voice,
1296
- });
1297
-
1298
- if (!ttsResult.success || !ttsResult.audioBuffer) {
1299
- ttsSpinner.fail(chalk.red(`Failed to generate narration: ${ttsResult.error || "Unknown error"}`));
1300
- process.exit(1);
1301
- }
1302
-
1303
- await writeFile(narrationPath, ttsResult.audioBuffer);
1304
- narrationDuration = await getAudioDuration(narrationPath);
1305
-
1306
- // Update segment duration in storyboard
1307
- segment.duration = narrationDuration;
1308
-
1309
- ttsSpinner.succeed(chalk.green(`Generated narration (${narrationDuration.toFixed(1)}s)`));
1310
- }
1311
-
1312
- // Step 2: Regenerate image if needed
1313
- const imagePath = resolve(outputDir, `scene-${sceneNum}.png`);
1314
-
1315
- if (regenerateImage && imageApiKey) {
1316
- const imageSpinner = ora(`🎨 Regenerating image for scene ${sceneNum}...`).start();
1317
-
1318
- const imageProvider = options.imageProvider || "gemini";
1319
-
1320
- // Build prompt with character description for consistency
1321
- const characterDesc = segment.characterDescription || segments[0]?.characterDescription;
1322
- let imagePrompt = segment.visualStyle
1323
- ? `${segment.visuals}. Style: ${segment.visualStyle}`
1324
- : segment.visuals;
1325
-
1326
- // Add character description to prompt if available
1327
- if (characterDesc) {
1328
- imagePrompt = `${imagePrompt}\n\nIMPORTANT - Character appearance must match exactly: ${characterDesc}`;
1329
- }
1330
-
1331
- // Check if we should use reference-based generation for character consistency
1332
- const refSceneNum = options.referenceScene ? parseInt(options.referenceScene) : null;
1333
- let referenceImageBuffer: Buffer | undefined;
1334
-
1335
- if (refSceneNum && refSceneNum >= 1 && refSceneNum <= segments.length && refSceneNum !== sceneNum) {
1336
- const refImagePath = resolve(outputDir, `scene-${refSceneNum}.png`);
1337
- if (existsSync(refImagePath)) {
1338
- referenceImageBuffer = await readFile(refImagePath);
1339
- imageSpinner.text = `🎨 Regenerating image for scene ${sceneNum} (using scene ${refSceneNum} as reference)...`;
1340
- }
1341
- } else if (!refSceneNum) {
1342
- // Auto-detect: use the first available scene image as reference
1343
- for (let i = 1; i <= segments.length; i++) {
1344
- if (i !== sceneNum) {
1345
- const otherImagePath = resolve(outputDir, `scene-${i}.png`);
1346
- if (existsSync(otherImagePath)) {
1347
- referenceImageBuffer = await readFile(otherImagePath);
1348
- imageSpinner.text = `🎨 Regenerating image for scene ${sceneNum} (using scene ${i} as reference)...`;
1349
- break;
1350
- }
1351
- }
1352
- }
1353
- }
1354
-
1355
- // Determine image size/aspect ratio based on provider
1356
- const dalleImageSizes: Record<string, "1536x1024" | "1024x1536" | "1024x1024"> = {
1357
- "16:9": "1536x1024",
1358
- "9:16": "1024x1536",
1359
- "1:1": "1024x1024",
1360
- };
1361
- let imageBuffer: Buffer | undefined;
1362
- let imageUrl: string | undefined;
1363
- let imageError: string | undefined;
1364
-
1365
- if (imageProvider === "openai" || imageProvider === "dalle") {
1366
- const openaiImage = new OpenAIImageProvider();
1367
- await openaiImage.initialize({ apiKey: imageApiKey });
1368
- const imageResult = await openaiImage.generateImage(imagePrompt, {
1369
- size: dalleImageSizes[options.aspectRatio] || "1536x1024",
1370
- quality: "standard",
1371
- });
1372
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
1373
- imageUrl = imageResult.images[0].url;
1374
- } else {
1375
- imageError = imageResult.error;
1376
- }
1377
- } else if (imageProvider === "gemini") {
1378
- const gemini = new GeminiProvider();
1379
- await gemini.initialize({ apiKey: imageApiKey });
1380
-
1381
- // Use editImage with reference for character consistency
1382
- if (referenceImageBuffer) {
1383
- // Extract the main action from the scene description (take first action if multiple)
1384
- const simplifiedVisuals = segment.visuals.split(/[,.]/).find((part: string) =>
1385
- part.includes("standing") || part.includes("sitting") || part.includes("walking") ||
1386
- part.includes("lying") || part.includes("reaching") || part.includes("looking") ||
1387
- part.includes("working") || part.includes("coding") || part.includes("typing")
1388
- ) || segment.visuals.split(".")[0];
1389
-
1390
- const editPrompt = `Generate a new image showing the SAME SINGLE person from the reference image in a new scene.
1391
-
1392
- REFERENCE: Look at the person in the reference image - their face, hair, build, and overall appearance.
1393
-
1394
- NEW SCENE: ${simplifiedVisuals}
1395
-
1396
- CRITICAL RULES:
1397
- 1. Show ONLY ONE person - the exact same individual from the reference image
1398
- 2. The person must have the IDENTICAL face, hair style, and body type
1399
- 3. Do NOT show multiple people or duplicate the character
1400
- 4. Create a single moment in time, one pose, one action
1401
- 5. Match the art style and quality of the reference image
1402
-
1403
- Generate the single-person scene image now.`;
1404
-
1405
- const imageResult = await gemini.editImage([referenceImageBuffer], editPrompt, {
1406
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
1407
- });
1408
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
1409
- const img = imageResult.images[0];
1410
- if (img.base64) {
1411
- imageBuffer = Buffer.from(img.base64, "base64");
1412
- }
1413
- } else {
1414
- imageError = imageResult.error;
1415
- }
1416
- } else {
1417
- // No reference image, use regular generation
1418
- const imageResult = await gemini.generateImage(imagePrompt, {
1419
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
1420
- });
1421
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
1422
- const img = imageResult.images[0];
1423
- if (img.base64) {
1424
- imageBuffer = Buffer.from(img.base64, "base64");
1425
- }
1426
- } else {
1427
- imageError = imageResult.error;
1428
- }
1429
- }
1430
- } else if (imageProvider === "grok") {
1431
- const { GrokProvider } = await import("@vibeframe/ai-providers");
1432
- const grok = new GrokProvider();
1433
- await grok.initialize({ apiKey: imageApiKey });
1434
- const imageResult = await grok.generateImage(imagePrompt, {
1435
- aspectRatio: options.aspectRatio || "16:9",
1436
- });
1437
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
1438
- const img = imageResult.images[0];
1439
- if (img.base64) {
1440
- imageBuffer = Buffer.from(img.base64, "base64");
1441
- } else if (img.url) {
1442
- imageUrl = img.url;
1443
- }
1444
- } else {
1445
- imageError = imageResult.error;
1446
- }
1447
- }
1448
-
1449
- if (imageBuffer) {
1450
- await writeFile(imagePath, imageBuffer);
1451
- imageSpinner.succeed(chalk.green("Generated image"));
1452
- } else if (imageUrl) {
1453
- const response = await fetch(imageUrl);
1454
- const buffer = Buffer.from(await response.arrayBuffer());
1455
- await writeFile(imagePath, buffer);
1456
- imageSpinner.succeed(chalk.green("Generated image"));
1457
- } else {
1458
- const errorMsg = imageError || "Unknown error";
1459
- imageSpinner.fail(chalk.red(`Failed to generate image: ${errorMsg}`));
1460
- process.exit(1);
1461
- }
1462
- }
1463
-
1464
- // Step 3: Regenerate video if needed
1465
- const videoPath = resolve(outputDir, `scene-${sceneNum}.mp4`);
1466
-
1467
- if (regenerateVideo && videoApiKey) {
1468
- const videoSpinner = ora(`🎬 Regenerating video for scene ${sceneNum}...`).start();
1469
-
1470
- // Check if image exists
1471
- if (!existsSync(imagePath)) {
1472
- videoSpinner.fail(chalk.red(`Reference image not found: ${imagePath}`));
1473
- console.error(chalk.dim("Generate an image first with --image-only or regenerate all assets"));
1474
- process.exit(1);
1475
- }
1476
-
1477
- const imageBuffer = await readFile(imagePath);
1478
- const ext = extname(imagePath).toLowerCase().slice(1);
1479
- const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
1480
- const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
1481
-
1482
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
1483
- const maxRetries = parseInt(options.retries) || DEFAULT_VIDEO_RETRIES;
1484
-
1485
- let videoGenerated = false;
1486
-
1487
- if (options.generator === "kling") {
1488
- const kling = new KlingProvider();
1489
- await kling.initialize({ apiKey: videoApiKey });
1490
-
1491
- if (!kling.isConfigured()) {
1492
- videoSpinner.fail(chalk.red("Invalid Kling API key format. Use ACCESS_KEY:SECRET_KEY"));
1493
- process.exit(1);
1494
- }
1495
-
1496
- // Try to use image-to-video if ImgBB API key is available
1497
- const imgbbApiKey = await getApiKeyFromConfig("imgbb") || process.env.IMGBB_API_KEY;
1498
- let imageUrl: string | undefined;
1499
-
1500
- if (imgbbApiKey) {
1501
- videoSpinner.text = `🎬 Uploading image to ImgBB...`;
1502
- const uploadResult = await uploadToImgbb(imageBuffer, imgbbApiKey);
1503
- if (uploadResult.success && uploadResult.url) {
1504
- imageUrl = uploadResult.url;
1505
- videoSpinner.text = `🎬 Starting image-to-video generation...`;
1506
- } else {
1507
- console.log(chalk.yellow(`\n ⚠ ImgBB upload failed, falling back to text-to-video`));
1508
- }
1509
- }
1510
-
1511
- const result = await generateVideoWithRetryKling(
1512
- kling,
1513
- segment,
1514
- {
1515
- duration: videoDuration,
1516
- aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
1517
- referenceImage: imageUrl, // Use uploaded URL for image-to-video
1518
- },
1519
- maxRetries
1520
- );
1521
-
1522
- if (result) {
1523
- videoSpinner.text = `🎬 Waiting for video to complete...`;
1524
-
1525
- for (let attempt = 0; attempt <= maxRetries && !videoGenerated; attempt++) {
1526
- try {
1527
- const waitResult = await kling.waitForCompletion(
1528
- result.taskId,
1529
- result.type,
1530
- (status) => {
1531
- videoSpinner.text = `🎬 Scene ${sceneNum}: ${status.status}...`;
1532
- },
1533
- 600000
1534
- );
1535
-
1536
- if (waitResult.status === "completed" && waitResult.videoUrl) {
1537
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
1538
- await writeFile(videoPath, buffer);
1539
-
1540
- // Extend video to match narration duration if needed
1541
- await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${sceneNum}`, {
1542
- kling,
1543
- videoId: waitResult.videoId,
1544
- onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
1545
- });
1546
-
1547
- videoGenerated = true;
1548
- } else if (attempt < maxRetries) {
1549
- videoSpinner.text = `🎬 Scene ${sceneNum}: Retry ${attempt + 1}/${maxRetries}...`;
1550
- await sleep(RETRY_DELAY_MS);
1551
- }
1552
- } catch (err) {
1553
- if (attempt >= maxRetries) {
1554
- throw err;
1555
- }
1556
- videoSpinner.text = `🎬 Scene ${sceneNum}: Error, retry ${attempt + 1}/${maxRetries}...`;
1557
- await sleep(RETRY_DELAY_MS);
1558
- }
1559
- }
1560
- }
1561
- } else {
1562
- // Runway
1563
- const runway = new RunwayProvider();
1564
- await runway.initialize({ apiKey: videoApiKey });
1565
-
1566
- const result = await generateVideoWithRetryRunway(
1567
- runway,
1568
- segment,
1569
- referenceImage,
1570
- {
1571
- duration: videoDuration,
1572
- aspectRatio: options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio as "16:9" | "9:16"),
1573
- },
1574
- maxRetries,
1575
- (msg) => {
1576
- videoSpinner.text = `🎬 Scene ${sceneNum}: ${msg}`;
1577
- }
1578
- );
1579
-
1580
- if (result) {
1581
- videoSpinner.text = `🎬 Waiting for video to complete...`;
1582
-
1583
- for (let attempt = 0; attempt <= maxRetries && !videoGenerated; attempt++) {
1584
- try {
1585
- const waitResult = await runway.waitForCompletion(
1586
- result.taskId,
1587
- (status) => {
1588
- const progress = status.progress !== undefined ? `${status.progress}%` : status.status;
1589
- videoSpinner.text = `🎬 Scene ${sceneNum}: ${progress}...`;
1590
- },
1591
- 300000
1592
- );
1593
-
1594
- if (waitResult.status === "completed" && waitResult.videoUrl) {
1595
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
1596
- await writeFile(videoPath, buffer);
1597
-
1598
- // Extend video to match narration duration if needed (Runway - no Kling extend)
1599
- await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${sceneNum}`, {
1600
- onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
1601
- });
1602
-
1603
- videoGenerated = true;
1604
- } else if (attempt < maxRetries) {
1605
- videoSpinner.text = `🎬 Scene ${sceneNum}: Retry ${attempt + 1}/${maxRetries}...`;
1606
- await sleep(RETRY_DELAY_MS);
1607
- }
1608
- } catch (err) {
1609
- if (attempt >= maxRetries) {
1610
- throw err;
1611
- }
1612
- videoSpinner.text = `🎬 Scene ${sceneNum}: Error, retry ${attempt + 1}/${maxRetries}...`;
1613
- await sleep(RETRY_DELAY_MS);
1614
- }
1615
- }
1616
- }
1617
- }
1618
-
1619
- if (videoGenerated) {
1620
- videoSpinner.succeed(chalk.green("Generated video"));
1621
- } else {
1622
- videoSpinner.fail(chalk.red("Failed to generate video after all retries"));
1623
- process.exit(1);
1624
- }
1625
- }
1626
-
1627
- // Step 4: Recalculate startTime for ALL segments and re-save storyboard
1628
- {
1629
- let currentTime = 0;
1630
- for (const seg of segments) {
1631
- seg.startTime = currentTime;
1632
- currentTime += seg.duration;
1633
- }
1634
- await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
1635
- console.log(chalk.dim(` → Updated storyboard: ${storyboardPath}`));
1636
- }
1637
-
1638
- // Step 5: Update project.vibe.json if it exists — update ALL clips' startTime/duration
1639
- if (existsSync(projectPath)) {
1640
- const updateSpinner = ora("📦 Updating project file...").start();
1641
-
1642
- try {
1643
- const projectContent = await readFile(projectPath, "utf-8");
1644
- const projectData = JSON.parse(projectContent) as ProjectFile;
1645
-
1646
- // Find and update the source for this scene
1647
- const sceneName = `Scene ${sceneNum}`;
1648
- const narrationName = `Narration ${sceneNum}`;
1649
-
1650
- // Update video/image source
1651
- const videoSource = projectData.state.sources.find((s) => s.name === sceneName);
1652
- if (videoSource) {
1653
- const hasVideo = existsSync(videoPath);
1654
- videoSource.url = hasVideo ? videoPath : imagePath;
1655
- videoSource.type = hasVideo ? "video" : "image";
1656
- videoSource.duration = segment.duration;
1657
- }
1658
-
1659
- // Update narration source
1660
- const narrationSource = projectData.state.sources.find((s) => s.name === narrationName);
1661
- if (narrationSource && regenerateNarration) {
1662
- narrationSource.duration = narrationDuration;
1663
- }
1664
-
1665
- // Update ALL clips' startTime and duration based on recalculated segments
1666
- for (const clip of projectData.state.clips) {
1667
- const source = projectData.state.sources.find((s) => s.id === clip.sourceId);
1668
- if (!source) continue;
1669
-
1670
- // Match source name to segment (e.g., "Scene 1" → segment 0, "Narration 2" → segment 1)
1671
- const sceneMatch = source.name.match(/^Scene (\d+)$/);
1672
- const narrationMatch = source.name.match(/^Narration (\d+)$/);
1673
- const segIdx = sceneMatch ? parseInt(sceneMatch[1]) - 1 : narrationMatch ? parseInt(narrationMatch[1]) - 1 : -1;
1674
-
1675
- if (segIdx >= 0 && segIdx < segments.length) {
1676
- const seg = segments[segIdx];
1677
- clip.startTime = seg.startTime;
1678
- clip.duration = seg.duration;
1679
- clip.sourceEndOffset = seg.duration;
1680
- // Also update the source duration to match segment
1681
- source.duration = seg.duration;
1682
- }
1683
- }
1684
-
1685
- await writeFile(projectPath, JSON.stringify(projectData, null, 2), "utf-8");
1686
- updateSpinner.succeed(chalk.green("Updated project file (all clips synced)"));
1687
- } catch (err) {
1688
- updateSpinner.warn(chalk.yellow(`Could not update project file: ${err}`));
1689
- }
1690
- }
1691
-
1692
- console.log(chalk.green(` ✓ Scene ${sceneNum} done`));
1693
- } // End of for loop over sceneNums
1694
-
1695
- // Final summary
1696
- console.log();
1697
- console.log(chalk.bold.green(`✅ ${sceneNums.length} scene${sceneNums.length > 1 ? "s" : ""} regenerated successfully!`));
1698
- console.log(chalk.dim("─".repeat(60)));
1699
- console.log();
1700
- console.log(chalk.dim("Next steps:"));
1701
- console.log(chalk.dim(` vibe export ${outputDir}/ -o final.mp4`));
1702
- console.log();
1703
- } catch (error) {
1704
- console.error(chalk.red("Scene regeneration failed"));
1705
- console.error(error);
1706
- process.exit(1);
1707
- }
1708
- });
1709
-
1710
- }