@vibeframe/cli 0.27.0 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/LICENSE +21 -0
  2. package/dist/agent/adapters/index.d.ts +1 -0
  3. package/dist/agent/adapters/index.d.ts.map +1 -1
  4. package/dist/agent/adapters/index.js +5 -0
  5. package/dist/agent/adapters/index.js.map +1 -1
  6. package/dist/agent/adapters/openrouter.d.ts +16 -0
  7. package/dist/agent/adapters/openrouter.d.ts.map +1 -0
  8. package/dist/agent/adapters/openrouter.js +100 -0
  9. package/dist/agent/adapters/openrouter.js.map +1 -0
  10. package/dist/agent/types.d.ts +1 -1
  11. package/dist/agent/types.d.ts.map +1 -1
  12. package/dist/commands/agent.d.ts.map +1 -1
  13. package/dist/commands/agent.js +3 -1
  14. package/dist/commands/agent.js.map +1 -1
  15. package/dist/commands/ai-edit-cli.d.ts.map +1 -1
  16. package/dist/commands/ai-edit-cli.js +18 -0
  17. package/dist/commands/ai-edit-cli.js.map +1 -1
  18. package/dist/commands/generate.js +14 -0
  19. package/dist/commands/generate.js.map +1 -1
  20. package/dist/commands/schema.d.ts +1 -0
  21. package/dist/commands/schema.d.ts.map +1 -1
  22. package/dist/commands/schema.js +122 -21
  23. package/dist/commands/schema.js.map +1 -1
  24. package/dist/commands/setup.js +5 -2
  25. package/dist/commands/setup.js.map +1 -1
  26. package/dist/config/schema.d.ts +2 -1
  27. package/dist/config/schema.d.ts.map +1 -1
  28. package/dist/config/schema.js +2 -0
  29. package/dist/config/schema.js.map +1 -1
  30. package/dist/index.js +0 -0
  31. package/package.json +16 -12
  32. package/.turbo/turbo-build.log +0 -4
  33. package/.turbo/turbo-lint.log +0 -21
  34. package/.turbo/turbo-test.log +0 -689
  35. package/src/agent/adapters/claude.ts +0 -143
  36. package/src/agent/adapters/gemini.ts +0 -159
  37. package/src/agent/adapters/index.ts +0 -61
  38. package/src/agent/adapters/ollama.ts +0 -231
  39. package/src/agent/adapters/openai.ts +0 -116
  40. package/src/agent/adapters/xai.ts +0 -119
  41. package/src/agent/index.ts +0 -251
  42. package/src/agent/memory/index.ts +0 -151
  43. package/src/agent/prompts/system.ts +0 -106
  44. package/src/agent/tools/ai-editing.ts +0 -845
  45. package/src/agent/tools/ai-generation.ts +0 -1073
  46. package/src/agent/tools/ai-pipeline.ts +0 -1055
  47. package/src/agent/tools/ai.ts +0 -21
  48. package/src/agent/tools/batch.ts +0 -429
  49. package/src/agent/tools/e2e.test.ts +0 -545
  50. package/src/agent/tools/export.ts +0 -184
  51. package/src/agent/tools/filesystem.ts +0 -237
  52. package/src/agent/tools/index.ts +0 -150
  53. package/src/agent/tools/integration.test.ts +0 -775
  54. package/src/agent/tools/media.ts +0 -697
  55. package/src/agent/tools/project.ts +0 -313
  56. package/src/agent/tools/timeline.ts +0 -951
  57. package/src/agent/types.ts +0 -68
  58. package/src/commands/agent.ts +0 -340
  59. package/src/commands/ai-analyze.ts +0 -429
  60. package/src/commands/ai-animated-caption.ts +0 -390
  61. package/src/commands/ai-audio.ts +0 -941
  62. package/src/commands/ai-broll.ts +0 -490
  63. package/src/commands/ai-edit-cli.ts +0 -658
  64. package/src/commands/ai-edit.ts +0 -1542
  65. package/src/commands/ai-fill-gaps.ts +0 -566
  66. package/src/commands/ai-helpers.ts +0 -65
  67. package/src/commands/ai-highlights.ts +0 -1303
  68. package/src/commands/ai-image.ts +0 -761
  69. package/src/commands/ai-motion.ts +0 -347
  70. package/src/commands/ai-narrate.ts +0 -451
  71. package/src/commands/ai-review.ts +0 -309
  72. package/src/commands/ai-script-pipeline-cli.ts +0 -1710
  73. package/src/commands/ai-script-pipeline.ts +0 -1365
  74. package/src/commands/ai-suggest-edit.ts +0 -264
  75. package/src/commands/ai-video-fx.ts +0 -445
  76. package/src/commands/ai-video.ts +0 -915
  77. package/src/commands/ai-viral.ts +0 -595
  78. package/src/commands/ai-visual-fx.ts +0 -601
  79. package/src/commands/ai.test.ts +0 -627
  80. package/src/commands/ai.ts +0 -307
  81. package/src/commands/analyze.ts +0 -282
  82. package/src/commands/audio.ts +0 -644
  83. package/src/commands/batch.test.ts +0 -279
  84. package/src/commands/batch.ts +0 -440
  85. package/src/commands/detect.ts +0 -329
  86. package/src/commands/doctor.ts +0 -237
  87. package/src/commands/edit-cmd.ts +0 -1014
  88. package/src/commands/export.ts +0 -918
  89. package/src/commands/generate.ts +0 -2146
  90. package/src/commands/media.ts +0 -177
  91. package/src/commands/output.ts +0 -142
  92. package/src/commands/pipeline.ts +0 -398
  93. package/src/commands/project.test.ts +0 -127
  94. package/src/commands/project.ts +0 -149
  95. package/src/commands/sanitize.ts +0 -60
  96. package/src/commands/schema.ts +0 -130
  97. package/src/commands/setup.ts +0 -509
  98. package/src/commands/timeline.test.ts +0 -499
  99. package/src/commands/timeline.ts +0 -529
  100. package/src/commands/validate.ts +0 -77
  101. package/src/config/config.test.ts +0 -197
  102. package/src/config/index.ts +0 -125
  103. package/src/config/schema.ts +0 -82
  104. package/src/engine/index.ts +0 -2
  105. package/src/engine/project.test.ts +0 -702
  106. package/src/engine/project.ts +0 -439
  107. package/src/index.ts +0 -146
  108. package/src/utils/api-key.test.ts +0 -41
  109. package/src/utils/api-key.ts +0 -247
  110. package/src/utils/audio.ts +0 -83
  111. package/src/utils/exec-safe.ts +0 -75
  112. package/src/utils/first-run.ts +0 -52
  113. package/src/utils/provider-resolver.ts +0 -56
  114. package/src/utils/remotion.ts +0 -951
  115. package/src/utils/subtitle.test.ts +0 -227
  116. package/src/utils/subtitle.ts +0 -169
  117. package/src/utils/tty.ts +0 -196
  118. package/tsconfig.json +0 -20
@@ -1,1365 +0,0 @@
1
- /**
2
- * @module ai-script-pipeline
3
- *
4
- * Script-to-video pipeline and scene regeneration execute functions.
5
- *
6
- * CLI commands: script-to-video, regenerate-scene
7
- *
8
- * Execute functions:
9
- * executeScriptToVideo - Full pipeline: storyboard -> TTS -> images -> videos -> project
10
- * executeRegenerateScene - Re-generate specific scene(s) in an existing project
11
- *
12
- * Also exports shared helpers: uploadToImgbb, extendVideoToTarget,
13
- * generateVideoWithRetryKling, generateVideoWithRetryRunway, generateVideoWithRetryVeo, waitForVideoWithRetry
14
- *
15
- * @dependencies Claude (storyboard), ElevenLabs (TTS), OpenAI/Gemini (images),
16
- * Kling/Runway (video), FFmpeg (assembly/extension)
17
- */
18
-
19
- import { readFile, writeFile, mkdir, unlink, rename } from "node:fs/promises";
20
- import { resolve, basename, extname } from "node:path";
21
- import { existsSync } from "node:fs";
22
- import chalk from "chalk";
23
- import {
24
- GeminiProvider,
25
- OpenAIProvider,
26
- OpenAIImageProvider,
27
- ClaudeProvider,
28
- ElevenLabsProvider,
29
- KlingProvider,
30
- RunwayProvider,
31
- GrokProvider,
32
- } from "@vibeframe/ai-providers";
33
- import { getApiKey } from "../utils/api-key.js";
34
- import { getApiKeyFromConfig } from "../config/index.js";
35
- import { Project } from "../engine/index.js";
36
- import { getAudioDuration, getVideoDuration, extendVideoNaturally } from "../utils/audio.js";
37
- import { applyTextOverlays, type TextOverlayStyle, type VideoReviewFeedback } from "./ai-edit.js";
38
- import { executeReview } from "./ai-review.js";
39
- import { execSafe } from "../utils/exec-safe.js";
40
- import { downloadVideo } from "./ai-helpers.js";
41
-
42
- /** A single scene segment from the Claude-generated storyboard. */
43
- export interface StoryboardSegment {
44
- /** 1-based scene index (assigned during generation) */
45
- index?: number;
46
- /** Narrative description of the scene */
47
- description: string;
48
- /** Visual prompt for image/video generation */
49
- visuals: string;
50
- /** Art style directive (e.g. "cinematic", "anime") */
51
- visualStyle?: string;
52
- /** Character appearance description for consistency */
53
- characterDescription?: string;
54
- /** Reference to previous scene for continuity */
55
- previousSceneLink?: string;
56
- /** Voiceover narration text */
57
- narration?: string;
58
- /** Audio direction (e.g. "upbeat music") */
59
- audio?: string;
60
- /** Text lines to overlay on the video */
61
- textOverlays?: string[];
62
- /** Scene duration in seconds (updated to match narration) */
63
- duration: number;
64
- /** Cumulative start time in seconds */
65
- startTime: number;
66
- }
67
-
68
- /** Default retry count for video generation API calls. */
69
- export const DEFAULT_VIDEO_RETRIES = 2;
70
- /** Delay between retries in milliseconds. */
71
- export const RETRY_DELAY_MS = 5000;
72
-
73
- /**
74
- * Sleep helper
75
- */
76
- export function sleep(ms: number): Promise<void> {
77
- return new Promise((resolve) => setTimeout(resolve, ms));
78
- }
79
-
80
- /**
81
- * Upload image to ImgBB and return the URL
82
- * Used for Kling v2.5/v2.6 image-to-video which requires URL (not base64)
83
- */
84
- export async function uploadToImgbb(
85
- imageBuffer: Buffer,
86
- apiKey: string
87
- ): Promise<{ success: boolean; url?: string; error?: string }> {
88
- try {
89
- const base64Image = imageBuffer.toString("base64");
90
-
91
- const formData = new URLSearchParams();
92
- formData.append("key", apiKey);
93
- formData.append("image", base64Image);
94
-
95
- const response = await fetch("https://api.imgbb.com/1/upload", {
96
- method: "POST",
97
- body: formData,
98
- });
99
-
100
- if (!response.ok) {
101
- return { success: false, error: `ImgBB API error (${response.status}): ${response.statusText}` };
102
- }
103
-
104
- const data = (await response.json()) as {
105
- success?: boolean;
106
- data?: { url?: string };
107
- error?: { message?: string };
108
- };
109
-
110
- if (data.success && data.data?.url) {
111
- return { success: true, url: data.data.url };
112
- } else {
113
- return { success: false, error: data.error?.message || "Upload failed" };
114
- }
115
- } catch (err) {
116
- return { success: false, error: String(err) };
117
- }
118
- }
119
-
120
- /**
121
- * Extend a video to target duration using Kling extend API when possible,
122
- * with fallback to FFmpeg-based extendVideoNaturally.
123
- *
124
- * When the extension ratio > 1.4 and a Kling provider + videoId are available,
125
- * uses the Kling video-extend API for natural continuation instead of freeze frames.
126
- */
127
- export async function extendVideoToTarget(
128
- videoPath: string,
129
- targetDuration: number,
130
- outputDir: string,
131
- sceneLabel: string,
132
- options?: {
133
- kling?: KlingProvider;
134
- videoId?: string;
135
- onProgress?: (message: string) => void;
136
- }
137
- ): Promise<void> {
138
- const actualDuration = await getVideoDuration(videoPath);
139
- if (actualDuration >= targetDuration - 0.1) return;
140
-
141
- const ratio = targetDuration / actualDuration;
142
- const extendedPath = resolve(outputDir, `${basename(videoPath, ".mp4")}-extended.mp4`);
143
-
144
- // Try Kling extend API for large gaps (ratio > 1.4) where freeze frames look bad
145
- if (ratio > 1.4 && options?.kling && options?.videoId) {
146
- try {
147
- options.onProgress?.(`${sceneLabel}: Extending via Kling API...`);
148
- const extendResult = await options.kling.extendVideo(options.videoId, {
149
- duration: "5",
150
- });
151
-
152
- if (extendResult.status !== "failed" && extendResult.id) {
153
- const waitResult = await options.kling.waitForExtendCompletion(
154
- extendResult.id,
155
- (status) => {
156
- options.onProgress?.(`${sceneLabel}: extend ${status.status}...`);
157
- },
158
- 600000
159
- );
160
-
161
- if (waitResult.status === "completed" && waitResult.videoUrl) {
162
- // Download extended video
163
- const extendedVideoPath = resolve(outputDir, `${basename(videoPath, ".mp4")}-kling-ext.mp4`);
164
- const buffer = await downloadVideo(waitResult.videoUrl);
165
- await writeFile(extendedVideoPath, buffer);
166
-
167
- // Concatenate original + extension
168
- const concatPath = resolve(outputDir, `${basename(videoPath, ".mp4")}-concat.mp4`);
169
- const listPath = resolve(outputDir, `${basename(videoPath, ".mp4")}-concat.txt`);
170
- await writeFile(listPath, `file '${videoPath}'\nfile '${extendedVideoPath}'`, "utf-8");
171
- await execSafe("ffmpeg", ["-y", "-f", "concat", "-safe", "0", "-i", listPath, "-c", "copy", concatPath]);
172
-
173
- // Trim to exact target duration if concatenated video is longer
174
- const concatDuration = await getVideoDuration(concatPath);
175
- if (concatDuration > targetDuration + 0.5) {
176
- await execSafe("ffmpeg", ["-y", "-i", concatPath, "-t", targetDuration.toFixed(2), "-c", "copy", extendedPath]);
177
- await unlink(concatPath);
178
- } else {
179
- await rename(concatPath, extendedPath);
180
- }
181
-
182
- // Cleanup temp files
183
- await unlink(extendedVideoPath).catch(() => {});
184
- await unlink(listPath).catch(() => {});
185
- await unlink(videoPath);
186
- await rename(extendedPath, videoPath);
187
- return;
188
- }
189
- }
190
- // If Kling extend failed, fall through to FFmpeg fallback
191
- options.onProgress?.(`${sceneLabel}: Kling extend failed, using FFmpeg fallback...`);
192
- } catch {
193
- options.onProgress?.(`${sceneLabel}: Kling extend error, using FFmpeg fallback...`);
194
- }
195
- }
196
-
197
- // FFmpeg-based fallback (slowdown + frame interpolation + freeze frame)
198
- await extendVideoNaturally(videoPath, targetDuration, extendedPath);
199
- await unlink(videoPath);
200
- await rename(extendedPath, videoPath);
201
- }
202
-
203
- /**
204
- * Generate video with retry logic for Kling provider
205
- * Supports image-to-video with URL (v2.5/v2.6 models)
206
- */
207
- export async function generateVideoWithRetryKling(
208
- kling: KlingProvider,
209
- segment: StoryboardSegment,
210
- options: {
211
- duration: 5 | 10;
212
- aspectRatio: "16:9" | "9:16" | "1:1";
213
- referenceImage?: string; // Optional: base64 or URL for image2video
214
- },
215
- maxRetries: number,
216
- onProgress?: (message: string) => void
217
- ): Promise<{ taskId: string; type: "text2video" | "image2video" } | null> {
218
- // Build detailed prompt from storyboard segment
219
- const prompt = segment.visualStyle
220
- ? `${segment.visuals}. Style: ${segment.visualStyle}`
221
- : segment.visuals;
222
-
223
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
224
- try {
225
- const result = await kling.generateVideo(prompt, {
226
- prompt,
227
- // Pass reference image (base64 or URL) - KlingProvider handles v1.5 fallback for base64
228
- referenceImage: options.referenceImage,
229
- duration: options.duration,
230
- aspectRatio: options.aspectRatio,
231
- mode: "std", // Use std mode for faster generation
232
- });
233
-
234
- if (result.status !== "failed" && result.id) {
235
- return {
236
- taskId: result.id,
237
- type: options.referenceImage ? "image2video" : "text2video",
238
- };
239
- }
240
-
241
- if (attempt < maxRetries) {
242
- onProgress?.(`⚠ Retry ${attempt + 1}/${maxRetries}...`);
243
- await sleep(RETRY_DELAY_MS);
244
- }
245
- } catch (err) {
246
- const errMsg = err instanceof Error ? err.message : String(err);
247
- if (attempt < maxRetries) {
248
- onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
249
- await sleep(RETRY_DELAY_MS);
250
- } else {
251
- // Log the final error on last attempt
252
- console.error(chalk.dim(`\n [Kling error: ${errMsg}]`));
253
- }
254
- }
255
- }
256
- return null;
257
- }
258
-
259
- /**
260
- * Generate video with retry logic for Runway provider
261
- */
262
- export async function generateVideoWithRetryRunway(
263
- runway: RunwayProvider,
264
- segment: StoryboardSegment,
265
- referenceImage: string,
266
- options: {
267
- duration: 5 | 10;
268
- aspectRatio: "16:9" | "9:16";
269
- },
270
- maxRetries: number,
271
- onProgress?: (message: string) => void
272
- ): Promise<{ taskId: string } | null> {
273
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
274
- try {
275
- const result = await runway.generateVideo(segment.visuals, {
276
- prompt: segment.visuals,
277
- referenceImage,
278
- duration: options.duration,
279
- aspectRatio: options.aspectRatio,
280
- });
281
-
282
- if (result.status !== "failed" && result.id) {
283
- return { taskId: result.id };
284
- }
285
-
286
- if (attempt < maxRetries) {
287
- onProgress?.(`⚠ Retry ${attempt + 1}/${maxRetries}...`);
288
- await sleep(RETRY_DELAY_MS);
289
- }
290
- } catch (err) {
291
- const errMsg = err instanceof Error ? err.message : String(err);
292
- if (attempt < maxRetries) {
293
- onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
294
- await sleep(RETRY_DELAY_MS);
295
- } else {
296
- console.error(chalk.dim(`\n [Runway error: ${errMsg}]`));
297
- }
298
- }
299
- }
300
- return null;
301
- }
302
-
303
- /**
304
- * Generate video with retry logic for Veo (Gemini) provider
305
- */
306
- export async function generateVideoWithRetryVeo(
307
- gemini: GeminiProvider,
308
- segment: StoryboardSegment,
309
- options: {
310
- duration: 4 | 6 | 8;
311
- aspectRatio: "16:9" | "9:16" | "1:1";
312
- referenceImage?: string;
313
- },
314
- maxRetries: number,
315
- onProgress?: (message: string) => void
316
- ): Promise<{ operationName: string } | null> {
317
- const prompt = segment.visualStyle
318
- ? `${segment.visuals}. Style: ${segment.visualStyle}`
319
- : segment.visuals;
320
-
321
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
322
- try {
323
- const result = await gemini.generateVideo(prompt, {
324
- prompt,
325
- referenceImage: options.referenceImage,
326
- duration: options.duration,
327
- aspectRatio: options.aspectRatio,
328
- model: "veo-3.1-fast-generate-preview",
329
- });
330
-
331
- if (result.status !== "failed" && result.id) {
332
- return { operationName: result.id };
333
- }
334
-
335
- if (attempt < maxRetries) {
336
- onProgress?.(`⚠ Retry ${attempt + 1}/${maxRetries}...`);
337
- await sleep(RETRY_DELAY_MS);
338
- }
339
- } catch (err) {
340
- const errMsg = err instanceof Error ? err.message : String(err);
341
- if (attempt < maxRetries) {
342
- onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
343
- await sleep(RETRY_DELAY_MS);
344
- } else {
345
- console.error(chalk.dim(`\n [Veo error: ${errMsg}]`));
346
- }
347
- }
348
- }
349
- return null;
350
- }
351
-
352
- /**
353
- * Wait for video completion with retry logic
354
- */
355
- export async function waitForVideoWithRetry(
356
- provider: KlingProvider | RunwayProvider,
357
- taskId: string,
358
- providerType: "kling" | "runway",
359
- maxRetries: number,
360
- onProgress?: (message: string) => void,
361
- timeout?: number
362
- ): Promise<{ videoUrl: string } | null> {
363
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
364
- try {
365
- let result;
366
- if (providerType === "kling") {
367
- result = await (provider as KlingProvider).waitForCompletion(
368
- taskId,
369
- "image2video",
370
- (status) => onProgress?.(status.status || "processing"),
371
- timeout || 600000
372
- );
373
- } else {
374
- result = await (provider as RunwayProvider).waitForCompletion(
375
- taskId,
376
- (status) => {
377
- const progress = status.progress !== undefined ? `${status.progress}%` : status.status;
378
- onProgress?.(progress || "processing");
379
- },
380
- timeout || 300000
381
- );
382
- }
383
-
384
- if (result.status === "completed" && result.videoUrl) {
385
- return { videoUrl: result.videoUrl };
386
- }
387
-
388
- // If failed, try resubmitting on next attempt
389
- if (attempt < maxRetries) {
390
- onProgress?.(`⚠ Failed, will need resubmission...`);
391
- return null; // Signal need for resubmission
392
- }
393
- } catch (err) {
394
- if (attempt < maxRetries) {
395
- onProgress?.(`⚠ Error waiting, retry ${attempt + 1}/${maxRetries}...`);
396
- await sleep(RETRY_DELAY_MS);
397
- }
398
- }
399
- }
400
- return null;
401
- }
402
-
403
- /** Options for {@link executeScriptToVideo}. */
404
- export interface ScriptToVideoOptions {
405
- /** Raw script or concept text for the video */
406
- script: string;
407
- /** Output directory (default: "script-video-output") */
408
- outputDir?: string;
409
- /** Target total duration in seconds */
410
- duration?: number;
411
- /** ElevenLabs voice name or ID */
412
- voice?: string;
413
- /** Video generation provider */
414
- generator?: "runway" | "kling" | "veo";
415
- /** Image generation provider */
416
- imageProvider?: "openai" | "dalle" | "gemini" | "grok";
417
- /** Video aspect ratio */
418
- aspectRatio?: "16:9" | "9:16" | "1:1";
419
- /** Stop after image generation (skip video) */
420
- imagesOnly?: boolean;
421
- /** Skip voiceover generation */
422
- noVoiceover?: boolean;
423
- /** Max retries per video generation call */
424
- retries?: number;
425
- /** Creativity level for storyboard generation: low (default, consistent) or high (varied, unexpected) */
426
- creativity?: "low" | "high";
427
- /** Provider for storyboard generation: claude (default), openai, or gemini */
428
- storyboardProvider?: "claude" | "openai" | "gemini";
429
- /** Skip text overlay step */
430
- noTextOverlay?: boolean;
431
- /** Text overlay style preset */
432
- textStyle?: TextOverlayStyle;
433
- /** Enable AI review after assembly */
434
- review?: boolean;
435
- /** Auto-apply fixable issues from review */
436
- reviewAutoApply?: boolean;
437
- }
438
-
439
- /**
440
- * Narration entry with segment tracking
441
- */
442
- export interface NarrationEntry {
443
- /** Path to the narration audio file (null if failed) */
444
- path: string | null;
445
- /** Duration in seconds */
446
- duration: number;
447
- /** Index of the segment this narration belongs to */
448
- segmentIndex: number;
449
- /** Whether the narration failed to generate */
450
- failed: boolean;
451
- /** Error message if failed */
452
- error?: string;
453
- }
454
-
455
- /** Result from {@link executeScriptToVideo}. */
456
- export interface ScriptToVideoResult {
457
- /** Whether the pipeline completed successfully */
458
- success: boolean;
459
- /** Absolute path to the output directory */
460
- outputDir: string;
461
- /** Total number of storyboard scenes */
462
- scenes: number;
463
- /** Path to the generated storyboard JSON */
464
- storyboardPath?: string;
465
- /** Path to the generated .vibe.json project file */
466
- projectPath?: string;
467
- /** @deprecated Use narrationEntries for proper segment tracking */
468
- narrations?: string[];
469
- /** Narration entries with segment index tracking */
470
- narrationEntries?: NarrationEntry[];
471
- /** Paths to generated scene images */
472
- images?: string[];
473
- /** Paths to generated scene videos */
474
- videos?: string[];
475
- /** Total video duration in seconds */
476
- totalDuration?: number;
477
- /** 1-indexed scene numbers that failed to generate */
478
- failedScenes?: number[];
479
- /** Failed narration scene numbers (1-indexed) */
480
- failedNarrations?: number[];
481
- /** Error message on failure */
482
- error?: string;
483
- /** Review feedback from Gemini (when --review is used) */
484
- reviewFeedback?: VideoReviewFeedback;
485
- /** List of auto-applied fixes (when --review-auto-apply is used) */
486
- appliedFixes?: string[];
487
- /** Path to reviewed/fixed video (when review auto-applied) */
488
- reviewedVideoPath?: string;
489
- }
490
-
491
- /**
492
- * Execute the full script-to-video pipeline programmatically.
493
- *
494
- * Pipeline stages:
495
- * 1. Generate storyboard with Claude
496
- * 2. Generate per-scene voiceovers with ElevenLabs TTS
497
- * 3. Generate scene images (OpenAI/Gemini)
498
- * 4. Generate scene videos (Kling/Runway) with extension to match narration
499
- * 4.5. Apply text overlays if present in storyboard
500
- * 5. Assemble .vibe.json project file
501
- * 6. Optional AI review and auto-fix (Gemini)
502
- *
503
- * @param options - Pipeline configuration
504
- * @returns Result with paths to all generated assets and project file
505
- */
506
- export async function executeScriptToVideo(
507
- options: ScriptToVideoOptions
508
- ): Promise<ScriptToVideoResult> {
509
- const outputDir = options.outputDir || "script-video-output";
510
-
511
- try {
512
- // Get storyboard provider API key
513
- const storyboardProvider = options.storyboardProvider || "claude";
514
- let storyboardApiKey: string | undefined;
515
-
516
- if (storyboardProvider === "openai") {
517
- storyboardApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
518
- if (!storyboardApiKey) {
519
- return { success: false, outputDir, scenes: 0, error: "OpenAI API key required for storyboard generation (--storyboard-provider openai). Run 'vibe setup' or set OPENAI_API_KEY in .env" };
520
- }
521
- } else if (storyboardProvider === "gemini") {
522
- storyboardApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
523
- if (!storyboardApiKey) {
524
- return { success: false, outputDir, scenes: 0, error: "Google API key required for storyboard generation (--storyboard-provider gemini). Run 'vibe setup' or set GOOGLE_API_KEY in .env" };
525
- }
526
- } else {
527
- // Default: Claude
528
- storyboardApiKey = (await getApiKey("ANTHROPIC_API_KEY", "Anthropic")) ?? undefined;
529
- if (!storyboardApiKey) {
530
- return { success: false, outputDir, scenes: 0, error: "Anthropic API key required for storyboard generation. Run 'vibe setup' or set ANTHROPIC_API_KEY in .env" };
531
- }
532
- }
533
-
534
- // Get image provider API key
535
- let imageApiKey: string | undefined;
536
- const imageProvider = options.imageProvider || "openai";
537
-
538
- if (imageProvider === "openai" || imageProvider === "dalle") {
539
- imageApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
540
- if (!imageApiKey) {
541
- return { success: false, outputDir, scenes: 0, error: "OpenAI API key required for image generation. Run 'vibe setup' or set OPENAI_API_KEY in .env" };
542
- }
543
- } else if (imageProvider === "gemini") {
544
- imageApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
545
- if (!imageApiKey) {
546
- return { success: false, outputDir, scenes: 0, error: "Google API key required for Gemini image generation. Run 'vibe setup' or set GOOGLE_API_KEY in .env" };
547
- }
548
- } else if (imageProvider === "grok") {
549
- imageApiKey = (await getApiKey("XAI_API_KEY", "xAI")) ?? undefined;
550
- if (!imageApiKey) {
551
- return { success: false, outputDir, scenes: 0, error: "xAI API key required for Grok image generation. Run 'vibe setup' or set XAI_API_KEY in .env" };
552
- }
553
- }
554
-
555
- let elevenlabsApiKey: string | undefined;
556
- if (!options.noVoiceover) {
557
- elevenlabsApiKey = (await getApiKey("ELEVENLABS_API_KEY", "ElevenLabs")) ?? undefined;
558
- if (!elevenlabsApiKey) {
559
- return { success: false, outputDir, scenes: 0, error: "ElevenLabs API key required for voiceover (or use noVoiceover option). Run 'vibe setup' or set ELEVENLABS_API_KEY in .env" };
560
- }
561
- }
562
-
563
- let videoApiKey: string | undefined;
564
- if (!options.imagesOnly) {
565
- if (options.generator === "kling") {
566
- videoApiKey = (await getApiKey("KLING_API_KEY", "Kling")) ?? undefined;
567
- if (!videoApiKey) {
568
- return { success: false, outputDir, scenes: 0, error: "Kling API key required (or use imagesOnly option). Run 'vibe setup' or set KLING_API_KEY in .env" };
569
- }
570
- } else if (options.generator === "veo") {
571
- videoApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
572
- if (!videoApiKey) {
573
- return { success: false, outputDir, scenes: 0, error: "Google API key required for Veo video generation (or use imagesOnly option). Run 'vibe setup' or set GOOGLE_API_KEY in .env" };
574
- }
575
- } else {
576
- videoApiKey = (await getApiKey("RUNWAY_API_SECRET", "Runway")) ?? undefined;
577
- if (!videoApiKey) {
578
- return { success: false, outputDir, scenes: 0, error: "Runway API key required (or use imagesOnly option). Run 'vibe setup' or set RUNWAY_API_SECRET in .env" };
579
- }
580
- }
581
- }
582
-
583
- // Create output directory
584
- const absOutputDir = resolve(process.cwd(), outputDir);
585
- if (!existsSync(absOutputDir)) {
586
- await mkdir(absOutputDir, { recursive: true });
587
- }
588
-
589
- // Step 1: Generate storyboard
590
- let segments: StoryboardSegment[];
591
- const creativityOpts = { creativity: options.creativity };
592
-
593
- if (storyboardProvider === "openai") {
594
- const openai = new OpenAIProvider();
595
- await openai.initialize({ apiKey: storyboardApiKey! });
596
- segments = await openai.analyzeContent(options.script, options.duration, creativityOpts);
597
- } else if (storyboardProvider === "gemini") {
598
- const gemini = new GeminiProvider();
599
- await gemini.initialize({ apiKey: storyboardApiKey! });
600
- segments = await gemini.analyzeContent(options.script, options.duration, creativityOpts);
601
- } else {
602
- const claude = new ClaudeProvider();
603
- await claude.initialize({ apiKey: storyboardApiKey! });
604
- segments = await claude.analyzeContent(options.script, options.duration, creativityOpts);
605
- }
606
-
607
- if (segments.length === 0) {
608
- return { success: false, outputDir, scenes: 0, error: "Failed to generate storyboard" };
609
- }
610
-
611
- // Save storyboard
612
- const storyboardPath = resolve(absOutputDir, "storyboard.json");
613
- await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
614
-
615
- const result: ScriptToVideoResult = {
616
- success: true,
617
- outputDir: absOutputDir,
618
- scenes: segments.length,
619
- storyboardPath,
620
- narrations: [],
621
- narrationEntries: [],
622
- images: [],
623
- videos: [],
624
- failedScenes: [],
625
- failedNarrations: [],
626
- };
627
-
628
- // Step 2: Generate per-scene voiceovers with ElevenLabs
629
- if (!options.noVoiceover && elevenlabsApiKey) {
630
- const elevenlabs = new ElevenLabsProvider();
631
- await elevenlabs.initialize({ apiKey: elevenlabsApiKey });
632
-
633
- for (let i = 0; i < segments.length; i++) {
634
- const segment = segments[i];
635
- const narrationText = segment.narration || segment.description;
636
-
637
- if (!narrationText) {
638
- // No narration text for this segment - add placeholder entry
639
- result.narrationEntries!.push({
640
- path: null,
641
- duration: segment.duration,
642
- segmentIndex: i,
643
- failed: false, // Not failed, just no text
644
- });
645
- continue;
646
- }
647
-
648
- const ttsResult = await elevenlabs.textToSpeech(narrationText, {
649
- voiceId: options.voice,
650
- });
651
-
652
- if (ttsResult.success && ttsResult.audioBuffer) {
653
- const audioPath = resolve(absOutputDir, `narration-${i + 1}.mp3`);
654
- await writeFile(audioPath, ttsResult.audioBuffer);
655
-
656
- // Get actual audio duration
657
- const actualDuration = await getAudioDuration(audioPath);
658
- segment.duration = actualDuration;
659
-
660
- // Add to both arrays for backwards compatibility
661
- result.narrations!.push(audioPath);
662
- result.narrationEntries!.push({
663
- path: audioPath,
664
- duration: actualDuration,
665
- segmentIndex: i,
666
- failed: false,
667
- });
668
- } else {
669
- // TTS failed - add placeholder entry with error info
670
- result.narrationEntries!.push({
671
- path: null,
672
- duration: segment.duration, // Keep original estimated duration
673
- segmentIndex: i,
674
- failed: true,
675
- error: ttsResult.error || "Unknown TTS error",
676
- });
677
- result.failedNarrations!.push(i + 1); // 1-indexed for user display
678
- }
679
- }
680
-
681
- // Recalculate startTime for all segments
682
- let currentTime = 0;
683
- for (const segment of segments) {
684
- segment.startTime = currentTime;
685
- currentTime += segment.duration;
686
- }
687
-
688
- // Re-save storyboard with updated durations
689
- await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
690
- }
691
-
692
- // Step 3: Generate images
693
- const dalleImageSizes: Record<string, "1536x1024" | "1024x1536" | "1024x1024"> = {
694
- "16:9": "1536x1024",
695
- "9:16": "1024x1536",
696
- "1:1": "1024x1024",
697
- };
698
- let openaiImageInstance: OpenAIImageProvider | undefined;
699
- let geminiInstance: GeminiProvider | undefined;
700
- let grokInstance: GrokProvider | undefined;
701
-
702
- if (imageProvider === "openai" || imageProvider === "dalle") {
703
- openaiImageInstance = new OpenAIImageProvider();
704
- await openaiImageInstance.initialize({ apiKey: imageApiKey! });
705
- } else if (imageProvider === "gemini") {
706
- geminiInstance = new GeminiProvider();
707
- await geminiInstance.initialize({ apiKey: imageApiKey! });
708
- } else if (imageProvider === "grok") {
709
- grokInstance = new GrokProvider();
710
- await grokInstance.initialize({ apiKey: imageApiKey! });
711
- }
712
-
713
- const imagePaths: string[] = [];
714
- for (let i = 0; i < segments.length; i++) {
715
- const segment = segments[i];
716
- const imagePrompt = segment.visualStyle
717
- ? `${segment.visuals}. Style: ${segment.visualStyle}`
718
- : segment.visuals;
719
-
720
- try {
721
- let imageBuffer: Buffer | undefined;
722
- let imageUrl: string | undefined;
723
-
724
- if ((imageProvider === "openai" || imageProvider === "dalle") && openaiImageInstance) {
725
- const imageResult = await openaiImageInstance.generateImage(imagePrompt, {
726
- size: dalleImageSizes[options.aspectRatio || "16:9"] || "1536x1024",
727
- quality: "standard",
728
- });
729
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
730
- // GPT Image 1.5 returns base64, DALL-E 3 returns URL
731
- const img = imageResult.images[0];
732
- if (img.base64) {
733
- imageBuffer = Buffer.from(img.base64, "base64");
734
- } else if (img.url) {
735
- imageUrl = img.url;
736
- }
737
- }
738
- // else: imageResult.error is available but not captured
739
- } else if (imageProvider === "gemini" && geminiInstance) {
740
- const imageResult = await geminiInstance.generateImage(imagePrompt, {
741
- aspectRatio: (options.aspectRatio || "16:9") as "16:9" | "9:16" | "1:1",
742
- });
743
- if (imageResult.success && imageResult.images?.[0]?.base64) {
744
- imageBuffer = Buffer.from(imageResult.images[0].base64, "base64");
745
- }
746
- // else: imageResult.error is available but not captured
747
- } else if (imageProvider === "grok" && grokInstance) {
748
- const imageResult = await grokInstance.generateImage(imagePrompt, {
749
- aspectRatio: options.aspectRatio || "16:9",
750
- });
751
- if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
752
- const img = imageResult.images[0];
753
- if (img.base64) {
754
- imageBuffer = Buffer.from(img.base64, "base64");
755
- } else if (img.url) {
756
- imageUrl = img.url;
757
- }
758
- }
759
- }
760
-
761
- const imagePath = resolve(absOutputDir, `scene-${i + 1}.png`);
762
- if (imageBuffer) {
763
- await writeFile(imagePath, imageBuffer);
764
- imagePaths.push(imagePath);
765
- result.images!.push(imagePath);
766
- } else if (imageUrl) {
767
- const response = await fetch(imageUrl);
768
- const buffer = Buffer.from(await response.arrayBuffer());
769
- await writeFile(imagePath, buffer);
770
- imagePaths.push(imagePath);
771
- result.images!.push(imagePath);
772
- } else {
773
- // Track failed scene - error details not captured (see provider imageResult.error)
774
- // The failedScenes array tracks which scenes failed for the caller
775
- imagePaths.push("");
776
- }
777
- } catch {
778
- imagePaths.push("");
779
- }
780
-
781
- // Rate limiting delay
782
- if (i < segments.length - 1) {
783
- await new Promise((r) => setTimeout(r, 500));
784
- }
785
- }
786
-
787
- // Step 4: Generate videos (if not images-only)
788
- const videoPaths: string[] = [];
789
- const maxRetries = options.retries ?? DEFAULT_VIDEO_RETRIES;
790
-
791
- if (!options.imagesOnly && videoApiKey) {
792
- if (options.generator === "kling") {
793
- const kling = new KlingProvider();
794
- await kling.initialize({ apiKey: videoApiKey });
795
-
796
- if (!kling.isConfigured()) {
797
- return { success: false, outputDir: absOutputDir, scenes: segments.length, error: "Invalid Kling API key format. Use ACCESS_KEY:SECRET_KEY" };
798
- }
799
-
800
- for (let i = 0; i < segments.length; i++) {
801
- if (!imagePaths[i]) {
802
- videoPaths.push("");
803
- continue;
804
- }
805
-
806
- const segment = segments[i] as StoryboardSegment;
807
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
808
-
809
- // Using text2video since Kling's image2video requires URL (not base64)
810
- const taskResult = await generateVideoWithRetryKling(
811
- kling,
812
- segment,
813
- { duration: videoDuration, aspectRatio: (options.aspectRatio || "16:9") as "16:9" | "9:16" | "1:1" },
814
- maxRetries
815
- );
816
-
817
- if (taskResult) {
818
- try {
819
- const waitResult = await kling.waitForCompletion(taskResult.taskId, taskResult.type, undefined, 600000);
820
- if (waitResult.status === "completed" && waitResult.videoUrl) {
821
- const videoPath = resolve(absOutputDir, `scene-${i + 1}.mp4`);
822
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
823
- await writeFile(videoPath, buffer);
824
-
825
- // Extend video to match narration duration if needed
826
- const targetDuration = segment.duration; // Already updated to narration length
827
- const actualVideoDuration = await getVideoDuration(videoPath);
828
-
829
- if (actualVideoDuration < targetDuration - 0.1) {
830
- const extendedPath = resolve(absOutputDir, `scene-${i + 1}-extended.mp4`);
831
- await extendVideoNaturally(videoPath, targetDuration, extendedPath);
832
- // Replace original with extended version
833
- await unlink(videoPath);
834
- await rename(extendedPath, videoPath);
835
- }
836
-
837
- videoPaths.push(videoPath);
838
- result.videos!.push(videoPath);
839
- } else {
840
- videoPaths.push("");
841
- result.failedScenes!.push(i + 1);
842
- }
843
- } catch {
844
- videoPaths.push("");
845
- result.failedScenes!.push(i + 1);
846
- }
847
- } else {
848
- videoPaths.push("");
849
- result.failedScenes!.push(i + 1);
850
- }
851
- }
852
- } else if (options.generator === "veo") {
853
- // Veo (Gemini)
854
- const veo = new GeminiProvider();
855
- await veo.initialize({ apiKey: videoApiKey });
856
-
857
- for (let i = 0; i < segments.length; i++) {
858
- if (!imagePaths[i]) {
859
- videoPaths.push("");
860
- continue;
861
- }
862
-
863
- const segment = segments[i] as StoryboardSegment;
864
- const veoDuration = (segment.duration > 6 ? 8 : segment.duration > 4 ? 6 : 4) as 4 | 6 | 8;
865
-
866
- const taskResult = await generateVideoWithRetryVeo(
867
- veo,
868
- segment,
869
- { duration: veoDuration, aspectRatio: (options.aspectRatio || "16:9") as "16:9" | "9:16" | "1:1" },
870
- maxRetries
871
- );
872
-
873
- if (taskResult) {
874
- try {
875
- const waitResult = await veo.waitForVideoCompletion(taskResult.operationName, undefined, 300000);
876
- if (waitResult.status === "completed" && waitResult.videoUrl) {
877
- const videoPath = resolve(absOutputDir, `scene-${i + 1}.mp4`);
878
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
879
- await writeFile(videoPath, buffer);
880
-
881
- // Extend video to match narration duration if needed
882
- const targetDuration = segment.duration;
883
- const actualVideoDuration = await getVideoDuration(videoPath);
884
-
885
- if (actualVideoDuration < targetDuration - 0.1) {
886
- const extendedPath = resolve(absOutputDir, `scene-${i + 1}-extended.mp4`);
887
- await extendVideoNaturally(videoPath, targetDuration, extendedPath);
888
- await unlink(videoPath);
889
- await rename(extendedPath, videoPath);
890
- }
891
-
892
- videoPaths.push(videoPath);
893
- result.videos!.push(videoPath);
894
- } else {
895
- videoPaths.push("");
896
- result.failedScenes!.push(i + 1);
897
- }
898
- } catch {
899
- videoPaths.push("");
900
- result.failedScenes!.push(i + 1);
901
- }
902
- } else {
903
- videoPaths.push("");
904
- result.failedScenes!.push(i + 1);
905
- }
906
- }
907
- } else {
908
- // Runway
909
- const runway = new RunwayProvider();
910
- await runway.initialize({ apiKey: videoApiKey });
911
-
912
- for (let i = 0; i < segments.length; i++) {
913
- if (!imagePaths[i]) {
914
- videoPaths.push("");
915
- continue;
916
- }
917
-
918
- const segment = segments[i] as StoryboardSegment;
919
- const imageBuffer = await readFile(imagePaths[i]);
920
- const ext = extname(imagePaths[i]).toLowerCase().slice(1);
921
- const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
922
- const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
923
-
924
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
925
- const aspectRatio = options.aspectRatio === "1:1" ? "16:9" : ((options.aspectRatio || "16:9") as "16:9" | "9:16");
926
-
927
- const taskResult = await generateVideoWithRetryRunway(
928
- runway,
929
- segment,
930
- referenceImage,
931
- { duration: videoDuration, aspectRatio },
932
- maxRetries
933
- );
934
-
935
- if (taskResult) {
936
- try {
937
- const waitResult = await runway.waitForCompletion(taskResult.taskId, undefined, 300000);
938
- if (waitResult.status === "completed" && waitResult.videoUrl) {
939
- const videoPath = resolve(absOutputDir, `scene-${i + 1}.mp4`);
940
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
941
- await writeFile(videoPath, buffer);
942
-
943
- // Extend video to match narration duration if needed
944
- const targetDuration = segment.duration;
945
- const actualVideoDuration = await getVideoDuration(videoPath);
946
-
947
- if (actualVideoDuration < targetDuration - 0.1) {
948
- const extendedPath = resolve(absOutputDir, `scene-${i + 1}-extended.mp4`);
949
- await extendVideoNaturally(videoPath, targetDuration, extendedPath);
950
- await unlink(videoPath);
951
- await rename(extendedPath, videoPath);
952
- }
953
-
954
- videoPaths.push(videoPath);
955
- result.videos!.push(videoPath);
956
- } else {
957
- videoPaths.push("");
958
- result.failedScenes!.push(i + 1);
959
- }
960
- } catch {
961
- videoPaths.push("");
962
- result.failedScenes!.push(i + 1);
963
- }
964
- } else {
965
- videoPaths.push("");
966
- result.failedScenes!.push(i + 1);
967
- }
968
- }
969
- }
970
- }
971
-
972
- // Step 4.5: Apply text overlays (if segments have textOverlays)
973
- if (!options.noTextOverlay) {
974
- for (let i = 0; i < segments.length; i++) {
975
- const segment = segments[i];
976
- if (segment.textOverlays && segment.textOverlays.length > 0 && videoPaths[i] && videoPaths[i] !== "") {
977
- try {
978
- const overlayOutput = videoPaths[i].replace(/(\.[^.]+)$/, "-overlay$1");
979
- const overlayResult = await applyTextOverlays({
980
- videoPath: videoPaths[i],
981
- texts: segment.textOverlays,
982
- outputPath: overlayOutput,
983
- style: options.textStyle || "lower-third",
984
- });
985
- if (overlayResult.success && overlayResult.outputPath) {
986
- videoPaths[i] = overlayResult.outputPath;
987
- }
988
- // Silent fallback: keep original on failure
989
- } catch {
990
- // Silent fallback: keep original video
991
- }
992
- }
993
- }
994
- }
995
-
996
- // Step 5: Create project file
997
- const project = new Project("Script-to-Video Output");
998
- project.setAspectRatio((options.aspectRatio || "16:9") as "16:9" | "9:16" | "1:1");
999
-
1000
- // Clear default tracks
1001
- const defaultTracks = project.getTracks();
1002
- for (const track of defaultTracks) {
1003
- project.removeTrack(track.id);
1004
- }
1005
-
1006
- const videoTrack = project.addTrack({
1007
- name: "Video",
1008
- type: "video",
1009
- order: 1,
1010
- isMuted: false,
1011
- isLocked: false,
1012
- isVisible: true,
1013
- });
1014
-
1015
- const audioTrack = project.addTrack({
1016
- name: "Audio",
1017
- type: "audio",
1018
- order: 0,
1019
- isMuted: false,
1020
- isLocked: false,
1021
- isVisible: true,
1022
- });
1023
-
1024
- // Add narration clips - use narrationEntries for proper segment alignment
1025
- if (result.narrationEntries && result.narrationEntries.length > 0) {
1026
- for (const entry of result.narrationEntries) {
1027
- // Skip failed or missing narrations
1028
- if (entry.failed || !entry.path) continue;
1029
-
1030
- const segment = segments[entry.segmentIndex];
1031
- const narrationDuration = await getAudioDuration(entry.path);
1032
-
1033
- const audioSource = project.addSource({
1034
- name: `Narration ${entry.segmentIndex + 1}`,
1035
- url: entry.path,
1036
- type: "audio",
1037
- duration: narrationDuration,
1038
- });
1039
-
1040
- project.addClip({
1041
- sourceId: audioSource.id,
1042
- trackId: audioTrack.id,
1043
- startTime: segment.startTime,
1044
- duration: narrationDuration,
1045
- sourceStartOffset: 0,
1046
- sourceEndOffset: narrationDuration,
1047
- });
1048
- }
1049
- }
1050
-
1051
- // Add video/image clips
1052
- let currentTime = 0;
1053
- for (let i = 0; i < segments.length; i++) {
1054
- const segment = segments[i];
1055
- const hasVideo = videoPaths[i] && videoPaths[i] !== "";
1056
- const hasImage = imagePaths[i] && imagePaths[i] !== "";
1057
-
1058
- if (!hasVideo && !hasImage) {
1059
- currentTime += segment.duration;
1060
- continue;
1061
- }
1062
-
1063
- const assetPath = hasVideo ? videoPaths[i] : imagePaths[i];
1064
- const mediaType = hasVideo ? "video" : "image";
1065
-
1066
- // Use actual video duration (after extension) instead of segment.duration
1067
- const actualDuration = hasVideo
1068
- ? await getVideoDuration(assetPath)
1069
- : segment.duration;
1070
-
1071
- const source = project.addSource({
1072
- name: `Scene ${i + 1}`,
1073
- url: assetPath,
1074
- type: mediaType as "video" | "image",
1075
- duration: actualDuration,
1076
- });
1077
-
1078
- project.addClip({
1079
- sourceId: source.id,
1080
- trackId: videoTrack.id,
1081
- startTime: currentTime,
1082
- duration: actualDuration,
1083
- sourceStartOffset: 0,
1084
- sourceEndOffset: actualDuration,
1085
- });
1086
-
1087
- currentTime += actualDuration;
1088
- }
1089
-
1090
- // Save project file
1091
- const projectPath = resolve(absOutputDir, "project.vibe.json");
1092
- await writeFile(projectPath, JSON.stringify(project.toJSON(), null, 2), "utf-8");
1093
- result.projectPath = projectPath;
1094
- result.totalDuration = currentTime;
1095
-
1096
- // Step 6: AI Review & Auto-fix (optional, --review flag)
1097
- if (options.review) {
1098
- try {
1099
- const storyboardFile = resolve(absOutputDir, "storyboard.json");
1100
- // Export project to temp MP4 for review (use first valid video as proxy)
1101
- const reviewTarget = videoPaths.find((p) => p && p !== "") || imagePaths.find((p) => p && p !== "");
1102
- if (reviewTarget) {
1103
- const reviewResult = await executeReview({
1104
- videoPath: reviewTarget,
1105
- storyboardPath: existsSync(storyboardFile) ? storyboardFile : undefined,
1106
- autoApply: options.reviewAutoApply,
1107
- model: "flash",
1108
- });
1109
-
1110
- if (reviewResult.success) {
1111
- result.reviewFeedback = reviewResult.feedback;
1112
- result.appliedFixes = reviewResult.appliedFixes;
1113
- result.reviewedVideoPath = reviewResult.outputPath;
1114
- }
1115
- }
1116
- } catch {
1117
- // Review is non-critical, continue with result
1118
- }
1119
- }
1120
-
1121
- return result;
1122
- } catch (error) {
1123
- return {
1124
- success: false,
1125
- outputDir,
1126
- scenes: 0,
1127
- error: error instanceof Error ? error.message : String(error),
1128
- };
1129
- }
1130
- }
1131
-
1132
- /** Options for {@link executeRegenerateScene}. */
1133
- export interface RegenerateSceneOptions {
1134
- /** Path to the project output directory containing storyboard.json */
1135
- projectDir: string;
1136
- /** 1-indexed scene numbers to regenerate */
1137
- scenes: number[];
1138
- /** Only regenerate video (keep existing image) */
1139
- videoOnly?: boolean;
1140
- /** Only regenerate narration audio */
1141
- narrationOnly?: boolean;
1142
- /** Only regenerate scene image */
1143
- imageOnly?: boolean;
1144
- /** Video generation provider */
1145
- generator?: "kling" | "runway" | "veo";
1146
- /** Image generation provider */
1147
- imageProvider?: "gemini" | "openai" | "grok";
1148
- /** ElevenLabs voice name or ID */
1149
- voice?: string;
1150
- /** Video aspect ratio */
1151
- aspectRatio?: "16:9" | "9:16" | "1:1";
1152
- /** Max retries per video generation call */
1153
- retries?: number;
1154
- /** Reference scene number for character consistency (auto-detects if not specified) */
1155
- referenceScene?: number;
1156
- }
1157
-
1158
- /** Result from {@link executeRegenerateScene}. */
1159
- export interface RegenerateSceneResult {
1160
- /** Whether all requested scenes were regenerated */
1161
- success: boolean;
1162
- /** 1-indexed scene numbers successfully regenerated */
1163
- regeneratedScenes: number[];
1164
- /** 1-indexed scene numbers that failed to regenerate */
1165
- failedScenes: number[];
1166
- /** Error message on failure */
1167
- error?: string;
1168
- }
1169
-
1170
- /**
1171
- * Regenerate specific scene(s) in an existing script-to-video project.
1172
- *
1173
- * Reads the storyboard.json from the project directory, then regenerates
1174
- * the requested scenes using the specified video/image provider. Supports
1175
- * image-to-video via ImgBB URL upload for Kling.
1176
- *
1177
- * @param options - Scene regeneration configuration
1178
- * @returns Result with lists of regenerated and failed scene numbers
1179
- */
1180
- export async function executeRegenerateScene(
1181
- options: RegenerateSceneOptions
1182
- ): Promise<RegenerateSceneResult> {
1183
- const result: RegenerateSceneResult = {
1184
- success: false,
1185
- regeneratedScenes: [],
1186
- failedScenes: [],
1187
- };
1188
-
1189
- try {
1190
- const outputDir = resolve(process.cwd(), options.projectDir);
1191
- const storyboardPath = resolve(outputDir, "storyboard.json");
1192
-
1193
- if (!existsSync(outputDir)) {
1194
- return { ...result, error: `Project directory not found: ${outputDir}` };
1195
- }
1196
-
1197
- if (!existsSync(storyboardPath)) {
1198
- return { ...result, error: `Storyboard not found: ${storyboardPath}` };
1199
- }
1200
-
1201
- const storyboardContent = await readFile(storyboardPath, "utf-8");
1202
- const segments: StoryboardSegment[] = JSON.parse(storyboardContent);
1203
-
1204
- // Validate scenes
1205
- for (const sceneNum of options.scenes) {
1206
- if (sceneNum < 1 || sceneNum > segments.length) {
1207
- return { ...result, error: `Scene ${sceneNum} does not exist. Storyboard has ${segments.length} scenes.` };
1208
- }
1209
- }
1210
-
1211
- const regenerateVideo = options.videoOnly || (!options.narrationOnly && !options.imageOnly);
1212
-
1213
- // Get API keys
1214
- let videoApiKey: string | undefined;
1215
- if (regenerateVideo) {
1216
- if (options.generator === "kling" || !options.generator) {
1217
- videoApiKey = (await getApiKey("KLING_API_KEY", "Kling")) ?? undefined;
1218
- if (!videoApiKey) {
1219
- return { ...result, error: "Kling API key required. Run 'vibe setup' or set KLING_API_KEY in .env" };
1220
- }
1221
- } else {
1222
- videoApiKey = (await getApiKey("RUNWAY_API_SECRET", "Runway")) ?? undefined;
1223
- if (!videoApiKey) {
1224
- return { ...result, error: "Runway API key required. Run 'vibe setup' or set RUNWAY_API_SECRET in .env" };
1225
- }
1226
- }
1227
- }
1228
-
1229
- // Process each scene
1230
- for (const sceneNum of options.scenes) {
1231
- const segment = segments[sceneNum - 1];
1232
- const imagePath = resolve(outputDir, `scene-${sceneNum}.png`);
1233
- const videoPath = resolve(outputDir, `scene-${sceneNum}.mp4`);
1234
-
1235
- if (regenerateVideo && videoApiKey) {
1236
- if (!existsSync(imagePath)) {
1237
- result.failedScenes.push(sceneNum);
1238
- continue;
1239
- }
1240
-
1241
- const imageBuffer = await readFile(imagePath);
1242
- const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
1243
- const maxRetries = options.retries ?? DEFAULT_VIDEO_RETRIES;
1244
-
1245
- if (options.generator === "kling" || !options.generator) {
1246
- const kling = new KlingProvider();
1247
- await kling.initialize({ apiKey: videoApiKey });
1248
-
1249
- if (!kling.isConfigured()) {
1250
- result.failedScenes.push(sceneNum);
1251
- continue;
1252
- }
1253
-
1254
- // Try to use image-to-video if ImgBB key available
1255
- const imgbbApiKey = await getApiKeyFromConfig("imgbb") || process.env.IMGBB_API_KEY;
1256
- let imageUrl: string | undefined;
1257
-
1258
- if (imgbbApiKey) {
1259
- const uploadResult = await uploadToImgbb(imageBuffer, imgbbApiKey);
1260
- if (uploadResult.success && uploadResult.url) {
1261
- imageUrl = uploadResult.url;
1262
- }
1263
- }
1264
-
1265
- const taskResult = await generateVideoWithRetryKling(
1266
- kling,
1267
- segment,
1268
- {
1269
- duration: videoDuration,
1270
- aspectRatio: (options.aspectRatio || "16:9") as "16:9" | "9:16" | "1:1",
1271
- referenceImage: imageUrl,
1272
- },
1273
- maxRetries
1274
- );
1275
-
1276
- if (taskResult) {
1277
- try {
1278
- const waitResult = await kling.waitForCompletion(taskResult.taskId, taskResult.type, undefined, 600000);
1279
- if (waitResult.status === "completed" && waitResult.videoUrl) {
1280
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
1281
- await writeFile(videoPath, buffer);
1282
-
1283
- // Extend video to match narration duration if needed
1284
- const targetDuration = segment.duration;
1285
- const actualVideoDuration = await getVideoDuration(videoPath);
1286
-
1287
- if (actualVideoDuration < targetDuration - 0.1) {
1288
- const extendedPath = resolve(outputDir, `scene-${sceneNum}-extended.mp4`);
1289
- await extendVideoNaturally(videoPath, targetDuration, extendedPath);
1290
- await unlink(videoPath);
1291
- await rename(extendedPath, videoPath);
1292
- }
1293
-
1294
- result.regeneratedScenes.push(sceneNum);
1295
- } else {
1296
- result.failedScenes.push(sceneNum);
1297
- }
1298
- } catch {
1299
- result.failedScenes.push(sceneNum);
1300
- }
1301
- } else {
1302
- result.failedScenes.push(sceneNum);
1303
- }
1304
- } else {
1305
- // Runway
1306
- const runway = new RunwayProvider();
1307
- await runway.initialize({ apiKey: videoApiKey });
1308
-
1309
- const ext = extname(imagePath).toLowerCase().slice(1);
1310
- const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
1311
- const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
1312
-
1313
- const aspectRatio = options.aspectRatio === "1:1" ? "16:9" : ((options.aspectRatio || "16:9") as "16:9" | "9:16");
1314
-
1315
- const taskResult = await generateVideoWithRetryRunway(
1316
- runway,
1317
- segment,
1318
- referenceImage,
1319
- { duration: videoDuration, aspectRatio },
1320
- maxRetries
1321
- );
1322
-
1323
- if (taskResult) {
1324
- try {
1325
- const waitResult = await runway.waitForCompletion(taskResult.taskId, undefined, 300000);
1326
- if (waitResult.status === "completed" && waitResult.videoUrl) {
1327
- const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
1328
- await writeFile(videoPath, buffer);
1329
-
1330
- // Extend video to match narration duration if needed
1331
- const targetDuration = segment.duration;
1332
- const actualVideoDuration = await getVideoDuration(videoPath);
1333
-
1334
- if (actualVideoDuration < targetDuration - 0.1) {
1335
- const extendedPath = resolve(outputDir, `scene-${sceneNum}-extended.mp4`);
1336
- await extendVideoNaturally(videoPath, targetDuration, extendedPath);
1337
- await unlink(videoPath);
1338
- await rename(extendedPath, videoPath);
1339
- }
1340
-
1341
- result.regeneratedScenes.push(sceneNum);
1342
- } else {
1343
- result.failedScenes.push(sceneNum);
1344
- }
1345
- } catch {
1346
- result.failedScenes.push(sceneNum);
1347
- }
1348
- } else {
1349
- result.failedScenes.push(sceneNum);
1350
- }
1351
- }
1352
- }
1353
- }
1354
-
1355
- result.success = result.failedScenes.length === 0;
1356
- return result;
1357
- } catch (error) {
1358
- return {
1359
- ...result,
1360
- error: error instanceof Error ? error.message : String(error),
1361
- };
1362
- }
1363
- }
1364
-
1365
- /* CLI command registration moved to ai-script-pipeline-cli.ts */