@vibeframe/cli 0.27.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/agent/adapters/index.d.ts +1 -0
- package/dist/agent/adapters/index.d.ts.map +1 -1
- package/dist/agent/adapters/index.js +5 -0
- package/dist/agent/adapters/index.js.map +1 -1
- package/dist/agent/adapters/openrouter.d.ts +16 -0
- package/dist/agent/adapters/openrouter.d.ts.map +1 -0
- package/dist/agent/adapters/openrouter.js +100 -0
- package/dist/agent/adapters/openrouter.js.map +1 -0
- package/dist/agent/types.d.ts +1 -1
- package/dist/agent/types.d.ts.map +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +3 -1
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/ai-edit-cli.d.ts.map +1 -1
- package/dist/commands/ai-edit-cli.js +18 -0
- package/dist/commands/ai-edit-cli.js.map +1 -1
- package/dist/commands/generate.js +14 -0
- package/dist/commands/generate.js.map +1 -1
- package/dist/commands/schema.d.ts +1 -0
- package/dist/commands/schema.d.ts.map +1 -1
- package/dist/commands/schema.js +122 -21
- package/dist/commands/schema.js.map +1 -1
- package/dist/commands/setup.js +5 -2
- package/dist/commands/setup.js.map +1 -1
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +2 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/index.js +0 -0
- package/package.json +16 -12
- package/.turbo/turbo-build.log +0 -4
- package/.turbo/turbo-lint.log +0 -21
- package/.turbo/turbo-test.log +0 -689
- package/src/agent/adapters/claude.ts +0 -143
- package/src/agent/adapters/gemini.ts +0 -159
- package/src/agent/adapters/index.ts +0 -61
- package/src/agent/adapters/ollama.ts +0 -231
- package/src/agent/adapters/openai.ts +0 -116
- package/src/agent/adapters/xai.ts +0 -119
- package/src/agent/index.ts +0 -251
- package/src/agent/memory/index.ts +0 -151
- package/src/agent/prompts/system.ts +0 -106
- package/src/agent/tools/ai-editing.ts +0 -845
- package/src/agent/tools/ai-generation.ts +0 -1073
- package/src/agent/tools/ai-pipeline.ts +0 -1055
- package/src/agent/tools/ai.ts +0 -21
- package/src/agent/tools/batch.ts +0 -429
- package/src/agent/tools/e2e.test.ts +0 -545
- package/src/agent/tools/export.ts +0 -184
- package/src/agent/tools/filesystem.ts +0 -237
- package/src/agent/tools/index.ts +0 -150
- package/src/agent/tools/integration.test.ts +0 -775
- package/src/agent/tools/media.ts +0 -697
- package/src/agent/tools/project.ts +0 -313
- package/src/agent/tools/timeline.ts +0 -951
- package/src/agent/types.ts +0 -68
- package/src/commands/agent.ts +0 -340
- package/src/commands/ai-analyze.ts +0 -429
- package/src/commands/ai-animated-caption.ts +0 -390
- package/src/commands/ai-audio.ts +0 -941
- package/src/commands/ai-broll.ts +0 -490
- package/src/commands/ai-edit-cli.ts +0 -658
- package/src/commands/ai-edit.ts +0 -1542
- package/src/commands/ai-fill-gaps.ts +0 -566
- package/src/commands/ai-helpers.ts +0 -65
- package/src/commands/ai-highlights.ts +0 -1303
- package/src/commands/ai-image.ts +0 -761
- package/src/commands/ai-motion.ts +0 -347
- package/src/commands/ai-narrate.ts +0 -451
- package/src/commands/ai-review.ts +0 -309
- package/src/commands/ai-script-pipeline-cli.ts +0 -1710
- package/src/commands/ai-script-pipeline.ts +0 -1365
- package/src/commands/ai-suggest-edit.ts +0 -264
- package/src/commands/ai-video-fx.ts +0 -445
- package/src/commands/ai-video.ts +0 -915
- package/src/commands/ai-viral.ts +0 -595
- package/src/commands/ai-visual-fx.ts +0 -601
- package/src/commands/ai.test.ts +0 -627
- package/src/commands/ai.ts +0 -307
- package/src/commands/analyze.ts +0 -282
- package/src/commands/audio.ts +0 -644
- package/src/commands/batch.test.ts +0 -279
- package/src/commands/batch.ts +0 -440
- package/src/commands/detect.ts +0 -329
- package/src/commands/doctor.ts +0 -237
- package/src/commands/edit-cmd.ts +0 -1014
- package/src/commands/export.ts +0 -918
- package/src/commands/generate.ts +0 -2146
- package/src/commands/media.ts +0 -177
- package/src/commands/output.ts +0 -142
- package/src/commands/pipeline.ts +0 -398
- package/src/commands/project.test.ts +0 -127
- package/src/commands/project.ts +0 -149
- package/src/commands/sanitize.ts +0 -60
- package/src/commands/schema.ts +0 -130
- package/src/commands/setup.ts +0 -509
- package/src/commands/timeline.test.ts +0 -499
- package/src/commands/timeline.ts +0 -529
- package/src/commands/validate.ts +0 -77
- package/src/config/config.test.ts +0 -197
- package/src/config/index.ts +0 -125
- package/src/config/schema.ts +0 -82
- package/src/engine/index.ts +0 -2
- package/src/engine/project.test.ts +0 -702
- package/src/engine/project.ts +0 -439
- package/src/index.ts +0 -146
- package/src/utils/api-key.test.ts +0 -41
- package/src/utils/api-key.ts +0 -247
- package/src/utils/audio.ts +0 -83
- package/src/utils/exec-safe.ts +0 -75
- package/src/utils/first-run.ts +0 -52
- package/src/utils/provider-resolver.ts +0 -56
- package/src/utils/remotion.ts +0 -951
- package/src/utils/subtitle.test.ts +0 -227
- package/src/utils/subtitle.ts +0 -169
- package/src/utils/tty.ts +0 -196
- package/tsconfig.json +0 -20
|
@@ -1,1710 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @module ai-script-pipeline-cli
|
|
3
|
-
* @description CLI command registration for the script-to-video pipeline and
|
|
4
|
-
* scene regeneration commands. Execute functions and helpers live in
|
|
5
|
-
* ai-script-pipeline.ts; this file wires them up as Commander.js subcommands.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import { Command } from "commander";
|
|
9
|
-
import { readFile, writeFile, mkdir, stat } from "node:fs/promises";
|
|
10
|
-
import { resolve, dirname, extname } from "node:path";
|
|
11
|
-
import { existsSync } from "node:fs";
|
|
12
|
-
import chalk from "chalk";
|
|
13
|
-
import ora from "ora";
|
|
14
|
-
import {
|
|
15
|
-
GeminiProvider,
|
|
16
|
-
OpenAIProvider,
|
|
17
|
-
OpenAIImageProvider,
|
|
18
|
-
ClaudeProvider,
|
|
19
|
-
ElevenLabsProvider,
|
|
20
|
-
KlingProvider,
|
|
21
|
-
RunwayProvider,
|
|
22
|
-
GrokProvider,
|
|
23
|
-
} from "@vibeframe/ai-providers";
|
|
24
|
-
import { getApiKey, loadEnv } from "../utils/api-key.js";
|
|
25
|
-
import { getApiKeyFromConfig } from "../config/index.js";
|
|
26
|
-
import { Project, type ProjectFile } from "../engine/index.js";
|
|
27
|
-
import { getAudioDuration } from "../utils/audio.js";
|
|
28
|
-
import { applyTextOverlays, type TextOverlayStyle } from "./ai-edit.js";
|
|
29
|
-
import { executeReview } from "./ai-review.js";
|
|
30
|
-
import {
|
|
31
|
-
type StoryboardSegment,
|
|
32
|
-
DEFAULT_VIDEO_RETRIES,
|
|
33
|
-
RETRY_DELAY_MS,
|
|
34
|
-
sleep,
|
|
35
|
-
uploadToImgbb,
|
|
36
|
-
extendVideoToTarget,
|
|
37
|
-
generateVideoWithRetryKling,
|
|
38
|
-
generateVideoWithRetryRunway,
|
|
39
|
-
} from "./ai-script-pipeline.js";
|
|
40
|
-
import { downloadVideo } from "./ai-helpers.js";
|
|
41
|
-
|
|
42
|
-
export function registerScriptPipelineCommands(aiCommand: Command): void {
|
|
43
|
-
// Script-to-Video command
|
|
44
|
-
aiCommand
|
|
45
|
-
.command("script-to-video")
|
|
46
|
-
.alias("s2v")
|
|
47
|
-
.description("Generate complete video from text script using AI pipeline")
|
|
48
|
-
.argument("<script>", "Script text or file path (use -f for file)")
|
|
49
|
-
.option("-f, --file", "Treat script argument as file path")
|
|
50
|
-
.option("-o, --output <path>", "Output project file path", "script-video.vibe.json")
|
|
51
|
-
.option("-d, --duration <seconds>", "Target total duration in seconds")
|
|
52
|
-
.option("-v, --voice <id>", "ElevenLabs voice ID for narration")
|
|
53
|
-
.option("-g, --generator <engine>", "Video generator: kling | runway | veo", "kling")
|
|
54
|
-
.option("-i, --image-provider <provider>", "Image provider: gemini | openai | grok", "gemini")
|
|
55
|
-
.option("-a, --aspect-ratio <ratio>", "Aspect ratio: 16:9 | 9:16 | 1:1", "16:9")
|
|
56
|
-
.option("--images-only", "Generate images only, skip video generation")
|
|
57
|
-
.option("--no-voiceover", "Skip voiceover generation")
|
|
58
|
-
.option("--output-dir <dir>", "Directory for generated assets", "script-video-output")
|
|
59
|
-
.option("--retries <count>", "Number of retries for video generation failures", String(DEFAULT_VIDEO_RETRIES))
|
|
60
|
-
.option("--sequential", "Generate videos one at a time (slower but more reliable)")
|
|
61
|
-
.option("--concurrency <count>", "Max concurrent video tasks in parallel mode (default: 3)", "3")
|
|
62
|
-
.option("-c, --creativity <level>", "Creativity level: low (default, consistent) or high (varied, unexpected)", "low")
|
|
63
|
-
.option("-s, --storyboard-provider <provider>", "Storyboard provider: claude (default), openai, or gemini", "claude")
|
|
64
|
-
.option("--no-text-overlay", "Skip text overlay step")
|
|
65
|
-
.option("--text-style <style>", "Text overlay style: lower-third, center-bold, subtitle, minimal", "lower-third")
|
|
66
|
-
.option("--review", "Run AI review after assembly (requires GOOGLE_API_KEY)")
|
|
67
|
-
.option("--review-auto-apply", "Auto-apply fixable issues from AI review")
|
|
68
|
-
.action(async (script: string, options) => {
|
|
69
|
-
try {
|
|
70
|
-
// Load environment variables from .env file
|
|
71
|
-
loadEnv();
|
|
72
|
-
|
|
73
|
-
// Get storyboard provider API key
|
|
74
|
-
const storyboardProvider = (options.storyboardProvider || "claude") as "claude" | "openai" | "gemini";
|
|
75
|
-
let storyboardApiKey: string | undefined;
|
|
76
|
-
|
|
77
|
-
if (storyboardProvider === "openai") {
|
|
78
|
-
storyboardApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
|
|
79
|
-
if (!storyboardApiKey) {
|
|
80
|
-
console.error(chalk.red("OpenAI API key required for storyboard generation (-s openai). Set OPENAI_API_KEY in .env or run: vibe setup"));
|
|
81
|
-
process.exit(1);
|
|
82
|
-
}
|
|
83
|
-
} else if (storyboardProvider === "gemini") {
|
|
84
|
-
storyboardApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
|
|
85
|
-
if (!storyboardApiKey) {
|
|
86
|
-
console.error(chalk.red("Google API key required for storyboard generation (-s gemini). Set GOOGLE_API_KEY in .env or run: vibe setup"));
|
|
87
|
-
process.exit(1);
|
|
88
|
-
}
|
|
89
|
-
} else if (storyboardProvider === "claude") {
|
|
90
|
-
storyboardApiKey = (await getApiKey("ANTHROPIC_API_KEY", "Anthropic")) ?? undefined;
|
|
91
|
-
if (!storyboardApiKey) {
|
|
92
|
-
console.error(chalk.red("Anthropic API key required for storyboard generation. Set ANTHROPIC_API_KEY in .env or run: vibe setup"));
|
|
93
|
-
process.exit(1);
|
|
94
|
-
}
|
|
95
|
-
} else {
|
|
96
|
-
console.error(chalk.red(`Unknown storyboard provider: ${storyboardProvider}. Use claude, openai, or gemini`));
|
|
97
|
-
process.exit(1);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// Get image provider API key
|
|
101
|
-
let imageApiKey: string | undefined;
|
|
102
|
-
const imageProvider = options.imageProvider || "openai";
|
|
103
|
-
|
|
104
|
-
if (imageProvider === "openai" || imageProvider === "dalle") {
|
|
105
|
-
imageApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
|
|
106
|
-
if (!imageApiKey) {
|
|
107
|
-
console.error(chalk.red("OpenAI API key required for DALL-E image generation. Set OPENAI_API_KEY in .env or run: vibe setup"));
|
|
108
|
-
process.exit(1);
|
|
109
|
-
}
|
|
110
|
-
} else if (imageProvider === "gemini") {
|
|
111
|
-
imageApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
|
|
112
|
-
if (!imageApiKey) {
|
|
113
|
-
console.error(chalk.red("Google API key required for Gemini image generation. Set GOOGLE_API_KEY in .env or run: vibe setup"));
|
|
114
|
-
process.exit(1);
|
|
115
|
-
}
|
|
116
|
-
} else if (imageProvider === "grok") {
|
|
117
|
-
imageApiKey = (await getApiKey("XAI_API_KEY", "xAI")) ?? undefined;
|
|
118
|
-
if (!imageApiKey) {
|
|
119
|
-
console.error(chalk.red("xAI API key required for Grok image generation. Set XAI_API_KEY in .env or run: vibe setup"));
|
|
120
|
-
process.exit(1);
|
|
121
|
-
}
|
|
122
|
-
} else {
|
|
123
|
-
console.error(chalk.red(`Unknown image provider: ${imageProvider}. Use openai, gemini, or grok`));
|
|
124
|
-
process.exit(1);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
let elevenlabsApiKey: string | undefined;
|
|
128
|
-
if (options.voiceover !== false) {
|
|
129
|
-
const key = await getApiKey("ELEVENLABS_API_KEY", "ElevenLabs");
|
|
130
|
-
if (!key) {
|
|
131
|
-
console.error(chalk.red("ElevenLabs API key required for voiceover (or use --no-voiceover). Set ELEVENLABS_API_KEY in .env or run: vibe setup"));
|
|
132
|
-
process.exit(1);
|
|
133
|
-
}
|
|
134
|
-
elevenlabsApiKey = key;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
let videoApiKey: string | undefined;
|
|
138
|
-
if (!options.imagesOnly) {
|
|
139
|
-
if (options.generator === "kling") {
|
|
140
|
-
const key = await getApiKey("KLING_API_KEY", "Kling");
|
|
141
|
-
if (!key) {
|
|
142
|
-
console.error(chalk.red("Kling API key required (or use --images-only). Set KLING_API_KEY in .env or run: vibe setup"));
|
|
143
|
-
process.exit(1);
|
|
144
|
-
}
|
|
145
|
-
videoApiKey = key;
|
|
146
|
-
} else {
|
|
147
|
-
const key = await getApiKey("RUNWAY_API_SECRET", "Runway");
|
|
148
|
-
if (!key) {
|
|
149
|
-
console.error(chalk.red("Runway API key required (or use --images-only). Set RUNWAY_API_SECRET in .env or run: vibe setup"));
|
|
150
|
-
process.exit(1);
|
|
151
|
-
}
|
|
152
|
-
videoApiKey = key;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// Read script content
|
|
157
|
-
let scriptContent = script;
|
|
158
|
-
if (options.file) {
|
|
159
|
-
const filePath = resolve(process.cwd(), script);
|
|
160
|
-
scriptContent = await readFile(filePath, "utf-8");
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// Determine output directory for assets
|
|
164
|
-
// If -o looks like a directory and --output-dir is not explicitly set, use -o directory for assets
|
|
165
|
-
let effectiveOutputDir = options.outputDir;
|
|
166
|
-
const outputLooksLikeDirectory =
|
|
167
|
-
options.output.endsWith("/") ||
|
|
168
|
-
(!options.output.endsWith(".json") && !options.output.endsWith(".vibe.json"));
|
|
169
|
-
|
|
170
|
-
if (outputLooksLikeDirectory && options.outputDir === "script-video-output") {
|
|
171
|
-
// User specified a directory for -o but didn't set --output-dir, use -o directory for assets
|
|
172
|
-
effectiveOutputDir = options.output;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Create output directory
|
|
176
|
-
const outputDir = resolve(process.cwd(), effectiveOutputDir);
|
|
177
|
-
if (!existsSync(outputDir)) {
|
|
178
|
-
await mkdir(outputDir, { recursive: true });
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// Validate creativity level
|
|
182
|
-
const creativity = options.creativity?.toLowerCase();
|
|
183
|
-
if (creativity && creativity !== "low" && creativity !== "high") {
|
|
184
|
-
console.error(chalk.red("Invalid creativity level. Use 'low' or 'high'."));
|
|
185
|
-
process.exit(1);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
console.log();
|
|
189
|
-
console.log(chalk.bold.cyan("🎬 Script-to-Video Pipeline"));
|
|
190
|
-
console.log(chalk.dim("─".repeat(60)));
|
|
191
|
-
if (creativity === "high") {
|
|
192
|
-
console.log(chalk.yellow("🎨 High creativity mode: Generating varied, unexpected scenes"));
|
|
193
|
-
}
|
|
194
|
-
console.log();
|
|
195
|
-
|
|
196
|
-
// Step 1: Generate storyboard
|
|
197
|
-
const providerLabel = storyboardProvider.charAt(0).toUpperCase() + storyboardProvider.slice(1);
|
|
198
|
-
const storyboardSpinnerText = creativity === "high"
|
|
199
|
-
? `Analyzing script with ${providerLabel} (high creativity)...`
|
|
200
|
-
: `Analyzing script with ${providerLabel}...`;
|
|
201
|
-
const storyboardSpinner = ora(storyboardSpinnerText).start();
|
|
202
|
-
|
|
203
|
-
let segments: StoryboardSegment[];
|
|
204
|
-
const creativityOpts = { creativity: creativity as "low" | "high" | undefined };
|
|
205
|
-
const durationOpt = options.duration ? parseFloat(options.duration) : undefined;
|
|
206
|
-
|
|
207
|
-
if (storyboardProvider === "openai") {
|
|
208
|
-
const openai = new OpenAIProvider();
|
|
209
|
-
await openai.initialize({ apiKey: storyboardApiKey! });
|
|
210
|
-
segments = await openai.analyzeContent(scriptContent, durationOpt, creativityOpts);
|
|
211
|
-
} else if (storyboardProvider === "gemini") {
|
|
212
|
-
const gemini = new GeminiProvider();
|
|
213
|
-
await gemini.initialize({ apiKey: storyboardApiKey! });
|
|
214
|
-
segments = await gemini.analyzeContent(scriptContent, durationOpt, creativityOpts);
|
|
215
|
-
} else {
|
|
216
|
-
const claude = new ClaudeProvider();
|
|
217
|
-
await claude.initialize({ apiKey: storyboardApiKey! });
|
|
218
|
-
segments = await claude.analyzeContent(scriptContent, durationOpt, creativityOpts);
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
if (segments.length === 0) {
|
|
222
|
-
storyboardSpinner.fail(chalk.red("Failed to generate storyboard (check API key and error above)"));
|
|
223
|
-
process.exit(1);
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
let totalDuration = segments.reduce((sum, seg) => sum + seg.duration, 0);
|
|
227
|
-
storyboardSpinner.succeed(chalk.green(`Generated ${segments.length} scenes (total: ${totalDuration}s)`));
|
|
228
|
-
|
|
229
|
-
// Save storyboard
|
|
230
|
-
const storyboardPath = resolve(outputDir, "storyboard.json");
|
|
231
|
-
await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
|
|
232
|
-
console.log(chalk.dim(` → Saved: ${storyboardPath}`));
|
|
233
|
-
console.log();
|
|
234
|
-
|
|
235
|
-
// Step 2: Generate per-scene voiceovers with ElevenLabs
|
|
236
|
-
const perSceneTTS: { path: string; duration: number; segmentIndex: number }[] = [];
|
|
237
|
-
const failedNarrations: { sceneNum: number; error: string }[] = [];
|
|
238
|
-
|
|
239
|
-
if (options.voiceover !== false && elevenlabsApiKey) {
|
|
240
|
-
const ttsSpinner = ora("🎙️ Generating voiceovers with ElevenLabs...").start();
|
|
241
|
-
|
|
242
|
-
const elevenlabs = new ElevenLabsProvider();
|
|
243
|
-
await elevenlabs.initialize({ apiKey: elevenlabsApiKey });
|
|
244
|
-
|
|
245
|
-
let totalCharacters = 0;
|
|
246
|
-
|
|
247
|
-
for (let i = 0; i < segments.length; i++) {
|
|
248
|
-
const segment = segments[i];
|
|
249
|
-
const narrationText = segment.narration || segment.description;
|
|
250
|
-
|
|
251
|
-
if (!narrationText) continue;
|
|
252
|
-
|
|
253
|
-
ttsSpinner.text = `🎙️ Generating narration ${i + 1}/${segments.length}...`;
|
|
254
|
-
|
|
255
|
-
let ttsResult = await elevenlabs.textToSpeech(narrationText, {
|
|
256
|
-
voiceId: options.voice,
|
|
257
|
-
});
|
|
258
|
-
|
|
259
|
-
if (!ttsResult.success || !ttsResult.audioBuffer) {
|
|
260
|
-
const errorMsg = ttsResult.error || "Unknown error";
|
|
261
|
-
failedNarrations.push({ sceneNum: i + 1, error: errorMsg });
|
|
262
|
-
ttsSpinner.text = `🎙️ Generating narration ${i + 1}/${segments.length}... (failed)`;
|
|
263
|
-
console.log(chalk.yellow(`\n ⚠ Narration ${i + 1} failed: ${errorMsg}`));
|
|
264
|
-
continue;
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
const audioPath = resolve(outputDir, `narration-${i + 1}.mp3`);
|
|
268
|
-
await writeFile(audioPath, ttsResult.audioBuffer);
|
|
269
|
-
|
|
270
|
-
// Get actual audio duration using ffprobe
|
|
271
|
-
let actualDuration = await getAudioDuration(audioPath);
|
|
272
|
-
|
|
273
|
-
// Auto speed-adjust if narration slightly exceeds video bracket (5s or 10s)
|
|
274
|
-
const videoBracket = segment.duration > 5 ? 10 : 5;
|
|
275
|
-
const overageRatio = actualDuration / videoBracket;
|
|
276
|
-
if (overageRatio > 1.0 && overageRatio <= 1.15) {
|
|
277
|
-
// Narration exceeds bracket by 0-15% — regenerate slightly faster
|
|
278
|
-
const adjustedSpeed = Math.min(1.2, parseFloat(overageRatio.toFixed(2)));
|
|
279
|
-
ttsSpinner.text = `🎙️ Narration ${i + 1}: adjusting speed to ${adjustedSpeed}x...`;
|
|
280
|
-
const speedResult = await elevenlabs.textToSpeech(narrationText, {
|
|
281
|
-
voiceId: options.voice,
|
|
282
|
-
speed: adjustedSpeed,
|
|
283
|
-
});
|
|
284
|
-
if (speedResult.success && speedResult.audioBuffer) {
|
|
285
|
-
await writeFile(audioPath, speedResult.audioBuffer);
|
|
286
|
-
actualDuration = await getAudioDuration(audioPath);
|
|
287
|
-
ttsResult = speedResult;
|
|
288
|
-
console.log(chalk.dim(` → Speed-adjusted narration ${i + 1}: ${adjustedSpeed}x → ${actualDuration.toFixed(1)}s`));
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
// Update segment duration to match actual narration length
|
|
293
|
-
segment.duration = actualDuration;
|
|
294
|
-
|
|
295
|
-
perSceneTTS.push({ path: audioPath, duration: actualDuration, segmentIndex: i });
|
|
296
|
-
totalCharacters += ttsResult.characterCount || 0;
|
|
297
|
-
|
|
298
|
-
console.log(chalk.dim(` → Saved: ${audioPath} (${actualDuration.toFixed(1)}s)`));
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
// Recalculate startTime for all segments based on updated durations
|
|
302
|
-
let currentTime = 0;
|
|
303
|
-
for (const segment of segments) {
|
|
304
|
-
segment.startTime = currentTime;
|
|
305
|
-
currentTime += segment.duration;
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
// Update total duration
|
|
309
|
-
totalDuration = segments.reduce((sum, seg) => sum + seg.duration, 0);
|
|
310
|
-
|
|
311
|
-
// Show success with failed count if any
|
|
312
|
-
if (failedNarrations.length > 0) {
|
|
313
|
-
ttsSpinner.warn(chalk.yellow(`Generated ${perSceneTTS.length}/${segments.length} narrations (${failedNarrations.length} failed)`));
|
|
314
|
-
} else {
|
|
315
|
-
ttsSpinner.succeed(chalk.green(`Generated ${perSceneTTS.length}/${segments.length} narrations (${totalCharacters} chars, ${totalDuration.toFixed(1)}s total)`));
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
// Re-save storyboard with updated durations
|
|
319
|
-
await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
|
|
320
|
-
console.log(chalk.dim(` → Updated storyboard: ${storyboardPath}`));
|
|
321
|
-
console.log();
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
// Step 3: Generate images with selected provider
|
|
325
|
-
const providerNames: Record<string, string> = {
|
|
326
|
-
openai: "OpenAI GPT Image 1.5",
|
|
327
|
-
dalle: "OpenAI GPT Image 1.5", // backward compatibility
|
|
328
|
-
gemini: "Gemini",
|
|
329
|
-
grok: "xAI Grok",
|
|
330
|
-
};
|
|
331
|
-
const imageSpinner = ora(`🎨 Generating visuals with ${providerNames[imageProvider]}...`).start();
|
|
332
|
-
|
|
333
|
-
// Determine image size/aspect ratio based on provider
|
|
334
|
-
const dalleImageSizes: Record<string, "1536x1024" | "1024x1536" | "1024x1024"> = {
|
|
335
|
-
"16:9": "1536x1024",
|
|
336
|
-
"9:16": "1024x1536",
|
|
337
|
-
"1:1": "1024x1024",
|
|
338
|
-
};
|
|
339
|
-
const imagePaths: string[] = [];
|
|
340
|
-
|
|
341
|
-
// Store first scene image for style continuity
|
|
342
|
-
let firstSceneImage: Buffer | undefined;
|
|
343
|
-
|
|
344
|
-
// Initialize the selected provider
|
|
345
|
-
let openaiImageInstance: OpenAIImageProvider | undefined;
|
|
346
|
-
let geminiInstance: GeminiProvider | undefined;
|
|
347
|
-
let grokInstance: GrokProvider | undefined;
|
|
348
|
-
|
|
349
|
-
if (imageProvider === "openai" || imageProvider === "dalle") {
|
|
350
|
-
openaiImageInstance = new OpenAIImageProvider();
|
|
351
|
-
await openaiImageInstance.initialize({ apiKey: imageApiKey });
|
|
352
|
-
} else if (imageProvider === "gemini") {
|
|
353
|
-
geminiInstance = new GeminiProvider();
|
|
354
|
-
await geminiInstance.initialize({ apiKey: imageApiKey });
|
|
355
|
-
} else if (imageProvider === "grok") {
|
|
356
|
-
grokInstance = new GrokProvider();
|
|
357
|
-
await grokInstance.initialize({ apiKey: imageApiKey });
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
// Get character description from first segment (should be same across all)
|
|
361
|
-
const characterDescription = segments[0]?.characterDescription;
|
|
362
|
-
|
|
363
|
-
for (let i = 0; i < segments.length; i++) {
|
|
364
|
-
const segment = segments[i];
|
|
365
|
-
imageSpinner.text = `🎨 Generating image ${i + 1}/${segments.length}: ${segment.description.slice(0, 30)}...`;
|
|
366
|
-
|
|
367
|
-
// Build comprehensive image prompt with character description
|
|
368
|
-
let imagePrompt = segment.visuals;
|
|
369
|
-
|
|
370
|
-
// Add character description to ensure consistency
|
|
371
|
-
if (characterDescription) {
|
|
372
|
-
imagePrompt = `CHARACTER (must match exactly): ${characterDescription}. SCENE: ${imagePrompt}`;
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
// Add visual style
|
|
376
|
-
if (segment.visualStyle) {
|
|
377
|
-
imagePrompt = `${imagePrompt}. STYLE: ${segment.visualStyle}`;
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// For scenes after the first, add extra continuity instruction (OpenAI)
|
|
381
|
-
// Gemini uses editImage with reference instead
|
|
382
|
-
if (i > 0 && firstSceneImage && imageProvider !== "gemini") {
|
|
383
|
-
imagePrompt = `${imagePrompt}. CRITICAL: The character must look IDENTICAL to the first scene - same face, hair, clothing, accessories.`;
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
try {
|
|
387
|
-
let imageBuffer: Buffer | undefined;
|
|
388
|
-
let imageUrl: string | undefined;
|
|
389
|
-
let imageError: string | undefined;
|
|
390
|
-
|
|
391
|
-
if ((imageProvider === "openai" || imageProvider === "dalle") && openaiImageInstance) {
|
|
392
|
-
const imageResult = await openaiImageInstance.generateImage(imagePrompt, {
|
|
393
|
-
size: dalleImageSizes[options.aspectRatio] || "1536x1024",
|
|
394
|
-
quality: "standard",
|
|
395
|
-
});
|
|
396
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
397
|
-
// GPT Image 1.5 returns base64, DALL-E 3 returns URL
|
|
398
|
-
const img = imageResult.images[0];
|
|
399
|
-
if (img.base64) {
|
|
400
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
401
|
-
} else if (img.url) {
|
|
402
|
-
imageUrl = img.url;
|
|
403
|
-
}
|
|
404
|
-
} else {
|
|
405
|
-
imageError = imageResult.error;
|
|
406
|
-
}
|
|
407
|
-
} else if (imageProvider === "gemini" && geminiInstance) {
|
|
408
|
-
// Gemini: use editImage with first scene reference for subsequent scenes
|
|
409
|
-
if (i > 0 && firstSceneImage) {
|
|
410
|
-
// Use editImage to maintain style continuity with first scene
|
|
411
|
-
const editPrompt = `Create a new scene for a video: ${imagePrompt}. IMPORTANT: Maintain the exact same character appearance, clothing, environment style, color palette, and art style as the reference image.`;
|
|
412
|
-
const imageResult = await geminiInstance.editImage([firstSceneImage], editPrompt, {
|
|
413
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
414
|
-
});
|
|
415
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
416
|
-
const img = imageResult.images[0];
|
|
417
|
-
if (img.base64) {
|
|
418
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
419
|
-
}
|
|
420
|
-
} else {
|
|
421
|
-
imageError = imageResult.error;
|
|
422
|
-
}
|
|
423
|
-
} else {
|
|
424
|
-
// First scene: use regular generateImage
|
|
425
|
-
const imageResult = await geminiInstance.generateImage(imagePrompt, {
|
|
426
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
427
|
-
});
|
|
428
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
429
|
-
const img = imageResult.images[0];
|
|
430
|
-
if (img.base64) {
|
|
431
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
432
|
-
}
|
|
433
|
-
} else {
|
|
434
|
-
imageError = imageResult.error;
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
} else if (imageProvider === "grok" && grokInstance) {
|
|
438
|
-
const imageResult = await grokInstance.generateImage(imagePrompt, {
|
|
439
|
-
aspectRatio: options.aspectRatio || "16:9",
|
|
440
|
-
});
|
|
441
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
442
|
-
const img = imageResult.images[0];
|
|
443
|
-
if (img.base64) {
|
|
444
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
445
|
-
} else if (img.url) {
|
|
446
|
-
imageUrl = img.url;
|
|
447
|
-
}
|
|
448
|
-
} else {
|
|
449
|
-
imageError = imageResult.error;
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
// Save the image
|
|
454
|
-
const imagePath = resolve(outputDir, `scene-${i + 1}.png`);
|
|
455
|
-
|
|
456
|
-
if (imageBuffer) {
|
|
457
|
-
await writeFile(imagePath, imageBuffer);
|
|
458
|
-
imagePaths.push(imagePath);
|
|
459
|
-
// Store first successful image for style continuity
|
|
460
|
-
if (!firstSceneImage) {
|
|
461
|
-
firstSceneImage = imageBuffer;
|
|
462
|
-
}
|
|
463
|
-
} else if (imageUrl) {
|
|
464
|
-
const response = await fetch(imageUrl);
|
|
465
|
-
const buffer = Buffer.from(await response.arrayBuffer());
|
|
466
|
-
await writeFile(imagePath, buffer);
|
|
467
|
-
imagePaths.push(imagePath);
|
|
468
|
-
// Store first successful image for style continuity
|
|
469
|
-
if (!firstSceneImage) {
|
|
470
|
-
firstSceneImage = buffer;
|
|
471
|
-
}
|
|
472
|
-
} else {
|
|
473
|
-
const errorMsg = imageError || "Unknown error";
|
|
474
|
-
console.log(chalk.yellow(`\n ⚠ Failed to generate image for scene ${i + 1}: ${errorMsg}`));
|
|
475
|
-
imagePaths.push("");
|
|
476
|
-
}
|
|
477
|
-
} catch (err) {
|
|
478
|
-
console.log(chalk.yellow(`\n ⚠ Error generating image for scene ${i + 1}: ${err}`));
|
|
479
|
-
imagePaths.push("");
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
// Small delay to avoid rate limiting
|
|
483
|
-
if (i < segments.length - 1) {
|
|
484
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
const successfulImages = imagePaths.filter((p) => p !== "").length;
|
|
489
|
-
imageSpinner.succeed(chalk.green(`Generated ${successfulImages}/${segments.length} images with ${providerNames[imageProvider]}`));
|
|
490
|
-
console.log();
|
|
491
|
-
|
|
492
|
-
// Step 4: Generate videos (if not images-only)
|
|
493
|
-
const videoPaths: string[] = [];
|
|
494
|
-
const failedScenes: number[] = []; // Track failed scenes for summary
|
|
495
|
-
const maxRetries = parseInt(options.retries) || DEFAULT_VIDEO_RETRIES;
|
|
496
|
-
|
|
497
|
-
if (!options.imagesOnly && videoApiKey) {
|
|
498
|
-
const videoSpinner = ora(`🎬 Generating videos with ${options.generator === "kling" ? "Kling" : "Runway"}...`).start();
|
|
499
|
-
|
|
500
|
-
if (options.generator === "kling") {
|
|
501
|
-
const kling = new KlingProvider();
|
|
502
|
-
await kling.initialize({ apiKey: videoApiKey });
|
|
503
|
-
|
|
504
|
-
if (!kling.isConfigured()) {
|
|
505
|
-
videoSpinner.fail(chalk.red("Invalid Kling API key format. Use ACCESS_KEY:SECRET_KEY"));
|
|
506
|
-
process.exit(1);
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
// Check for ImgBB API key for image-to-video support (from config or env)
|
|
510
|
-
const imgbbApiKey = await getApiKeyFromConfig("imgbb") || process.env.IMGBB_API_KEY;
|
|
511
|
-
const useImageToVideo = !!imgbbApiKey;
|
|
512
|
-
|
|
513
|
-
if (useImageToVideo) {
|
|
514
|
-
videoSpinner.text = `🎬 Uploading images to ImgBB for image-to-video...`;
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
// Upload images to ImgBB if API key is available (for Kling v2.x image-to-video)
|
|
518
|
-
const imageUrls: (string | undefined)[] = [];
|
|
519
|
-
if (useImageToVideo) {
|
|
520
|
-
for (let i = 0; i < imagePaths.length; i++) {
|
|
521
|
-
if (imagePaths[i] && imagePaths[i] !== "") {
|
|
522
|
-
try {
|
|
523
|
-
const imageBuffer = await readFile(imagePaths[i]);
|
|
524
|
-
const uploadResult = await uploadToImgbb(imageBuffer, imgbbApiKey);
|
|
525
|
-
if (uploadResult.success && uploadResult.url) {
|
|
526
|
-
imageUrls[i] = uploadResult.url;
|
|
527
|
-
} else {
|
|
528
|
-
console.log(chalk.yellow(`\n ⚠ Failed to upload image ${i + 1}: ${uploadResult.error}`));
|
|
529
|
-
imageUrls[i] = undefined;
|
|
530
|
-
}
|
|
531
|
-
} catch {
|
|
532
|
-
imageUrls[i] = undefined;
|
|
533
|
-
}
|
|
534
|
-
} else {
|
|
535
|
-
imageUrls[i] = undefined;
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
const uploadedCount = imageUrls.filter((u) => u).length;
|
|
539
|
-
if (uploadedCount > 0) {
|
|
540
|
-
videoSpinner.text = `🎬 Uploaded ${uploadedCount}/${imagePaths.length} images to ImgBB`;
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
// Sequential mode: generate one video at a time (slower but more reliable)
|
|
545
|
-
if (options.sequential) {
|
|
546
|
-
for (let i = 0; i < segments.length; i++) {
|
|
547
|
-
const segment = segments[i] as StoryboardSegment;
|
|
548
|
-
videoSpinner.text = `🎬 Scene ${i + 1}/${segments.length}: Starting...`;
|
|
549
|
-
|
|
550
|
-
const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
|
|
551
|
-
const referenceImage = imageUrls[i];
|
|
552
|
-
|
|
553
|
-
let completed = false;
|
|
554
|
-
for (let attempt = 0; attempt <= maxRetries && !completed; attempt++) {
|
|
555
|
-
const result = await generateVideoWithRetryKling(
|
|
556
|
-
kling,
|
|
557
|
-
segment,
|
|
558
|
-
{
|
|
559
|
-
duration: videoDuration,
|
|
560
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
561
|
-
referenceImage,
|
|
562
|
-
},
|
|
563
|
-
0, // Handle retries at this level
|
|
564
|
-
(msg) => {
|
|
565
|
-
videoSpinner.text = `🎬 Scene ${i + 1}/${segments.length}: ${msg}`;
|
|
566
|
-
}
|
|
567
|
-
);
|
|
568
|
-
|
|
569
|
-
if (!result) {
|
|
570
|
-
if (attempt < maxRetries) {
|
|
571
|
-
videoSpinner.text = `🎬 Scene ${i + 1}: Submit failed, retry ${attempt + 1}/${maxRetries}...`;
|
|
572
|
-
await sleep(RETRY_DELAY_MS);
|
|
573
|
-
continue;
|
|
574
|
-
}
|
|
575
|
-
console.log(chalk.yellow(`\n ⚠ Failed to start video generation for scene ${i + 1}`));
|
|
576
|
-
videoPaths[i] = "";
|
|
577
|
-
failedScenes.push(i + 1);
|
|
578
|
-
break;
|
|
579
|
-
}
|
|
580
|
-
|
|
581
|
-
try {
|
|
582
|
-
const waitResult = await kling.waitForCompletion(
|
|
583
|
-
result.taskId,
|
|
584
|
-
result.type,
|
|
585
|
-
(status) => {
|
|
586
|
-
videoSpinner.text = `🎬 Scene ${i + 1}/${segments.length}: ${status.status}...`;
|
|
587
|
-
},
|
|
588
|
-
600000
|
|
589
|
-
);
|
|
590
|
-
|
|
591
|
-
if (waitResult.status === "completed" && waitResult.videoUrl) {
|
|
592
|
-
const videoPath = resolve(outputDir, `scene-${i + 1}.mp4`);
|
|
593
|
-
const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
|
|
594
|
-
await writeFile(videoPath, buffer);
|
|
595
|
-
|
|
596
|
-
// Extend video to match narration duration if needed
|
|
597
|
-
await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${i + 1}`, {
|
|
598
|
-
kling,
|
|
599
|
-
videoId: waitResult.videoId,
|
|
600
|
-
onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
|
|
601
|
-
});
|
|
602
|
-
|
|
603
|
-
videoPaths[i] = videoPath;
|
|
604
|
-
completed = true;
|
|
605
|
-
console.log(chalk.green(`\n ✓ Scene ${i + 1} completed`));
|
|
606
|
-
} else if (attempt < maxRetries) {
|
|
607
|
-
videoSpinner.text = `🎬 Scene ${i + 1}: Failed, retry ${attempt + 1}/${maxRetries}...`;
|
|
608
|
-
await sleep(RETRY_DELAY_MS);
|
|
609
|
-
} else {
|
|
610
|
-
videoPaths[i] = "";
|
|
611
|
-
failedScenes.push(i + 1);
|
|
612
|
-
}
|
|
613
|
-
} catch (err) {
|
|
614
|
-
if (attempt < maxRetries) {
|
|
615
|
-
videoSpinner.text = `🎬 Scene ${i + 1}: Error, retry ${attempt + 1}/${maxRetries}...`;
|
|
616
|
-
await sleep(RETRY_DELAY_MS);
|
|
617
|
-
} else {
|
|
618
|
-
console.log(chalk.yellow(`\n ⚠ Error for scene ${i + 1}: ${err}`));
|
|
619
|
-
videoPaths[i] = "";
|
|
620
|
-
failedScenes.push(i + 1);
|
|
621
|
-
}
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
}
|
|
625
|
-
} else {
|
|
626
|
-
// Parallel mode (default): batch-based submission respecting concurrency limit
|
|
627
|
-
const concurrency = Math.max(1, parseInt(options.concurrency) || 3);
|
|
628
|
-
|
|
629
|
-
for (let batchStart = 0; batchStart < segments.length; batchStart += concurrency) {
|
|
630
|
-
const batchEnd = Math.min(batchStart + concurrency, segments.length);
|
|
631
|
-
const batchNum = Math.floor(batchStart / concurrency) + 1;
|
|
632
|
-
const totalBatches = Math.ceil(segments.length / concurrency);
|
|
633
|
-
|
|
634
|
-
if (totalBatches > 1) {
|
|
635
|
-
videoSpinner.text = `🎬 Batch ${batchNum}/${totalBatches}: submitting scenes ${batchStart + 1}-${batchEnd}...`;
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
// Phase 1: Submit batch
|
|
639
|
-
const tasks: Array<{ taskId: string; index: number; segment: StoryboardSegment; type: "text2video" | "image2video" }> = [];
|
|
640
|
-
|
|
641
|
-
for (let i = batchStart; i < batchEnd; i++) {
|
|
642
|
-
const segment = segments[i] as StoryboardSegment;
|
|
643
|
-
videoSpinner.text = `🎬 Submitting video task ${i + 1}/${segments.length}...`;
|
|
644
|
-
|
|
645
|
-
const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
|
|
646
|
-
const referenceImage = imageUrls[i];
|
|
647
|
-
|
|
648
|
-
const result = await generateVideoWithRetryKling(
|
|
649
|
-
kling,
|
|
650
|
-
segment,
|
|
651
|
-
{
|
|
652
|
-
duration: videoDuration,
|
|
653
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
654
|
-
referenceImage,
|
|
655
|
-
},
|
|
656
|
-
maxRetries,
|
|
657
|
-
(msg) => {
|
|
658
|
-
videoSpinner.text = `🎬 Scene ${i + 1}: ${msg}`;
|
|
659
|
-
}
|
|
660
|
-
);
|
|
661
|
-
|
|
662
|
-
if (result) {
|
|
663
|
-
tasks.push({ taskId: result.taskId, index: i, segment, type: result.type });
|
|
664
|
-
if (!videoPaths[i]) videoPaths[i] = "";
|
|
665
|
-
} else {
|
|
666
|
-
console.log(chalk.yellow(`\n ⚠ Failed to start video generation for scene ${i + 1} (after ${maxRetries} retries)`));
|
|
667
|
-
videoPaths[i] = "";
|
|
668
|
-
failedScenes.push(i + 1);
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
// Phase 2: Wait for batch completion
|
|
673
|
-
videoSpinner.text = `🎬 Waiting for batch ${batchNum} (${tasks.length} video${tasks.length > 1 ? "s" : ""})...`;
|
|
674
|
-
|
|
675
|
-
for (const task of tasks) {
|
|
676
|
-
let completed = false;
|
|
677
|
-
let currentTaskId = task.taskId;
|
|
678
|
-
let currentType = task.type;
|
|
679
|
-
|
|
680
|
-
for (let attempt = 0; attempt <= maxRetries && !completed; attempt++) {
|
|
681
|
-
try {
|
|
682
|
-
const result = await kling.waitForCompletion(
|
|
683
|
-
currentTaskId,
|
|
684
|
-
currentType,
|
|
685
|
-
(status) => {
|
|
686
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: ${status.status}...`;
|
|
687
|
-
},
|
|
688
|
-
600000
|
|
689
|
-
);
|
|
690
|
-
|
|
691
|
-
if (result.status === "completed" && result.videoUrl) {
|
|
692
|
-
const videoPath = resolve(outputDir, `scene-${task.index + 1}.mp4`);
|
|
693
|
-
const buffer = await downloadVideo(result.videoUrl, videoApiKey);
|
|
694
|
-
await writeFile(videoPath, buffer);
|
|
695
|
-
|
|
696
|
-
// Extend video to match narration duration if needed
|
|
697
|
-
await extendVideoToTarget(videoPath, task.segment.duration, outputDir, `Scene ${task.index + 1}`, {
|
|
698
|
-
kling,
|
|
699
|
-
videoId: result.videoId,
|
|
700
|
-
onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
|
|
701
|
-
});
|
|
702
|
-
|
|
703
|
-
videoPaths[task.index] = videoPath;
|
|
704
|
-
completed = true;
|
|
705
|
-
} else if (attempt < maxRetries) {
|
|
706
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: Retry ${attempt + 1}/${maxRetries}...`;
|
|
707
|
-
await sleep(RETRY_DELAY_MS);
|
|
708
|
-
|
|
709
|
-
const videoDuration = (task.segment.duration > 5 ? 10 : 5) as 5 | 10;
|
|
710
|
-
const retryReferenceImage = imageUrls[task.index];
|
|
711
|
-
|
|
712
|
-
const retryResult = await generateVideoWithRetryKling(
|
|
713
|
-
kling,
|
|
714
|
-
task.segment,
|
|
715
|
-
{
|
|
716
|
-
duration: videoDuration,
|
|
717
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
718
|
-
referenceImage: retryReferenceImage,
|
|
719
|
-
},
|
|
720
|
-
0
|
|
721
|
-
);
|
|
722
|
-
|
|
723
|
-
if (retryResult) {
|
|
724
|
-
currentTaskId = retryResult.taskId;
|
|
725
|
-
currentType = retryResult.type;
|
|
726
|
-
} else {
|
|
727
|
-
videoPaths[task.index] = "";
|
|
728
|
-
failedScenes.push(task.index + 1);
|
|
729
|
-
completed = true;
|
|
730
|
-
}
|
|
731
|
-
} else {
|
|
732
|
-
videoPaths[task.index] = "";
|
|
733
|
-
failedScenes.push(task.index + 1);
|
|
734
|
-
}
|
|
735
|
-
} catch (err) {
|
|
736
|
-
if (attempt >= maxRetries) {
|
|
737
|
-
console.log(chalk.yellow(`\n ⚠ Error completing video for scene ${task.index + 1}: ${err}`));
|
|
738
|
-
videoPaths[task.index] = "";
|
|
739
|
-
failedScenes.push(task.index + 1);
|
|
740
|
-
} else {
|
|
741
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: Error, retry ${attempt + 1}/${maxRetries}...`;
|
|
742
|
-
await sleep(RETRY_DELAY_MS);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
if (totalBatches > 1 && batchEnd < segments.length) {
|
|
749
|
-
console.log(chalk.dim(` → Batch ${batchNum}/${totalBatches} complete`));
|
|
750
|
-
}
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
} else {
|
|
754
|
-
// Runway
|
|
755
|
-
const runway = new RunwayProvider();
|
|
756
|
-
await runway.initialize({ apiKey: videoApiKey });
|
|
757
|
-
|
|
758
|
-
// Submit all video generation tasks with retry logic
|
|
759
|
-
const tasks: Array<{ taskId: string; index: number; imagePath: string; referenceImage: string; segment: StoryboardSegment }> = [];
|
|
760
|
-
|
|
761
|
-
for (let i = 0; i < segments.length; i++) {
|
|
762
|
-
if (!imagePaths[i]) {
|
|
763
|
-
videoPaths.push("");
|
|
764
|
-
continue;
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
const segment = segments[i] as StoryboardSegment;
|
|
768
|
-
videoSpinner.text = `🎬 Submitting video task ${i + 1}/${segments.length}...`;
|
|
769
|
-
|
|
770
|
-
const imageBuffer = await readFile(imagePaths[i]);
|
|
771
|
-
const ext = extname(imagePaths[i]).toLowerCase().slice(1);
|
|
772
|
-
const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
|
|
773
|
-
const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
|
|
774
|
-
|
|
775
|
-
// Use 10s video if narration > 5s to avoid video ending before narration
|
|
776
|
-
const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
|
|
777
|
-
|
|
778
|
-
const result = await generateVideoWithRetryRunway(
|
|
779
|
-
runway,
|
|
780
|
-
segment,
|
|
781
|
-
referenceImage,
|
|
782
|
-
{
|
|
783
|
-
duration: videoDuration,
|
|
784
|
-
aspectRatio: options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio as "16:9" | "9:16"),
|
|
785
|
-
},
|
|
786
|
-
maxRetries,
|
|
787
|
-
(msg) => {
|
|
788
|
-
videoSpinner.text = `🎬 Scene ${i + 1}: ${msg}`;
|
|
789
|
-
}
|
|
790
|
-
);
|
|
791
|
-
|
|
792
|
-
if (result) {
|
|
793
|
-
tasks.push({ taskId: result.taskId, index: i, imagePath: imagePaths[i], referenceImage, segment });
|
|
794
|
-
} else {
|
|
795
|
-
console.log(chalk.yellow(`\n ⚠ Failed to start video generation for scene ${i + 1} (after ${maxRetries} retries)`));
|
|
796
|
-
videoPaths[i] = "";
|
|
797
|
-
failedScenes.push(i + 1);
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
// Wait for all tasks to complete with retry logic
|
|
802
|
-
videoSpinner.text = `🎬 Waiting for ${tasks.length} video(s) to complete...`;
|
|
803
|
-
|
|
804
|
-
for (const task of tasks) {
|
|
805
|
-
let completed = false;
|
|
806
|
-
let currentTaskId = task.taskId;
|
|
807
|
-
|
|
808
|
-
for (let attempt = 0; attempt <= maxRetries && !completed; attempt++) {
|
|
809
|
-
try {
|
|
810
|
-
const result = await runway.waitForCompletion(
|
|
811
|
-
currentTaskId,
|
|
812
|
-
(status) => {
|
|
813
|
-
const progress = status.progress !== undefined ? `${status.progress}%` : status.status;
|
|
814
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: ${progress}...`;
|
|
815
|
-
},
|
|
816
|
-
300000 // 5 minute timeout per video
|
|
817
|
-
);
|
|
818
|
-
|
|
819
|
-
if (result.status === "completed" && result.videoUrl) {
|
|
820
|
-
const videoPath = resolve(outputDir, `scene-${task.index + 1}.mp4`);
|
|
821
|
-
const buffer = await downloadVideo(result.videoUrl, videoApiKey);
|
|
822
|
-
await writeFile(videoPath, buffer);
|
|
823
|
-
|
|
824
|
-
// Extend video to match narration duration if needed
|
|
825
|
-
await extendVideoToTarget(videoPath, task.segment.duration, outputDir, `Scene ${task.index + 1}`, {
|
|
826
|
-
onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
|
|
827
|
-
});
|
|
828
|
-
|
|
829
|
-
videoPaths[task.index] = videoPath;
|
|
830
|
-
completed = true;
|
|
831
|
-
} else if (attempt < maxRetries) {
|
|
832
|
-
// Resubmit task on failure
|
|
833
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: Retry ${attempt + 1}/${maxRetries}...`;
|
|
834
|
-
await sleep(RETRY_DELAY_MS);
|
|
835
|
-
|
|
836
|
-
const videoDuration = (task.segment.duration > 5 ? 10 : 5) as 5 | 10;
|
|
837
|
-
const retryResult = await generateVideoWithRetryRunway(
|
|
838
|
-
runway,
|
|
839
|
-
task.segment,
|
|
840
|
-
task.referenceImage,
|
|
841
|
-
{
|
|
842
|
-
duration: videoDuration,
|
|
843
|
-
aspectRatio: options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio as "16:9" | "9:16"),
|
|
844
|
-
},
|
|
845
|
-
0, // No nested retries
|
|
846
|
-
(msg) => {
|
|
847
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: ${msg}`;
|
|
848
|
-
}
|
|
849
|
-
);
|
|
850
|
-
|
|
851
|
-
if (retryResult) {
|
|
852
|
-
currentTaskId = retryResult.taskId;
|
|
853
|
-
} else {
|
|
854
|
-
videoPaths[task.index] = "";
|
|
855
|
-
failedScenes.push(task.index + 1);
|
|
856
|
-
completed = true; // Exit retry loop
|
|
857
|
-
}
|
|
858
|
-
} else {
|
|
859
|
-
videoPaths[task.index] = "";
|
|
860
|
-
failedScenes.push(task.index + 1);
|
|
861
|
-
}
|
|
862
|
-
} catch (err) {
|
|
863
|
-
if (attempt >= maxRetries) {
|
|
864
|
-
console.log(chalk.yellow(`\n ⚠ Error completing video for scene ${task.index + 1}: ${err}`));
|
|
865
|
-
videoPaths[task.index] = "";
|
|
866
|
-
failedScenes.push(task.index + 1);
|
|
867
|
-
} else {
|
|
868
|
-
videoSpinner.text = `🎬 Scene ${task.index + 1}: Error, retry ${attempt + 1}/${maxRetries}...`;
|
|
869
|
-
await sleep(RETRY_DELAY_MS);
|
|
870
|
-
}
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
const successfulVideos = videoPaths.filter((p) => p && p !== "").length;
|
|
877
|
-
videoSpinner.succeed(chalk.green(`Generated ${successfulVideos}/${segments.length} videos`));
|
|
878
|
-
console.log();
|
|
879
|
-
}
|
|
880
|
-
|
|
881
|
-
// Step 4.5: Apply text overlays (if segments have textOverlays)
|
|
882
|
-
if (options.textOverlay !== false) {
|
|
883
|
-
const overlaySegments = segments.filter(
|
|
884
|
-
(s: StoryboardSegment, i: number) => s.textOverlays && s.textOverlays.length > 0 && videoPaths[i] && videoPaths[i] !== ""
|
|
885
|
-
);
|
|
886
|
-
if (overlaySegments.length > 0) {
|
|
887
|
-
const overlaySpinner = ora(`Applying text overlays to ${overlaySegments.length} scene(s)...`).start();
|
|
888
|
-
let overlayCount = 0;
|
|
889
|
-
for (let i = 0; i < segments.length; i++) {
|
|
890
|
-
const segment = segments[i] as StoryboardSegment;
|
|
891
|
-
if (segment.textOverlays && segment.textOverlays.length > 0 && videoPaths[i] && videoPaths[i] !== "") {
|
|
892
|
-
try {
|
|
893
|
-
const overlayOutput = videoPaths[i].replace(/(\.[^.]+)$/, "-overlay$1");
|
|
894
|
-
const overlayResult = await applyTextOverlays({
|
|
895
|
-
videoPath: videoPaths[i],
|
|
896
|
-
texts: segment.textOverlays,
|
|
897
|
-
outputPath: overlayOutput,
|
|
898
|
-
style: (options.textStyle as TextOverlayStyle) || "lower-third",
|
|
899
|
-
});
|
|
900
|
-
if (overlayResult.success && overlayResult.outputPath) {
|
|
901
|
-
videoPaths[i] = overlayResult.outputPath;
|
|
902
|
-
overlayCount++;
|
|
903
|
-
}
|
|
904
|
-
} catch {
|
|
905
|
-
// Silent fallback: keep original video
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
overlaySpinner.succeed(chalk.green(`Applied text overlays to ${overlayCount} scene(s)`));
|
|
910
|
-
console.log();
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
// Step 5: Assemble project
|
|
915
|
-
const assembleSpinner = ora("Assembling project...").start();
|
|
916
|
-
|
|
917
|
-
const project = new Project("Script-to-Video Output");
|
|
918
|
-
project.setAspectRatio(options.aspectRatio as "16:9" | "9:16" | "1:1");
|
|
919
|
-
|
|
920
|
-
// Clear default tracks and create new ones
|
|
921
|
-
const defaultTracks = project.getTracks();
|
|
922
|
-
for (const track of defaultTracks) {
|
|
923
|
-
project.removeTrack(track.id);
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
const videoTrack = project.addTrack({
|
|
927
|
-
name: "Video",
|
|
928
|
-
type: "video",
|
|
929
|
-
order: 1,
|
|
930
|
-
isMuted: false,
|
|
931
|
-
isLocked: false,
|
|
932
|
-
isVisible: true,
|
|
933
|
-
});
|
|
934
|
-
|
|
935
|
-
const audioTrack = project.addTrack({
|
|
936
|
-
name: "Audio",
|
|
937
|
-
type: "audio",
|
|
938
|
-
order: 0,
|
|
939
|
-
isMuted: false,
|
|
940
|
-
isLocked: false,
|
|
941
|
-
isVisible: true,
|
|
942
|
-
});
|
|
943
|
-
|
|
944
|
-
// Add per-scene narration sources and clips
|
|
945
|
-
for (const tts of perSceneTTS) {
|
|
946
|
-
const segment = segments[tts.segmentIndex];
|
|
947
|
-
const audioSource = project.addSource({
|
|
948
|
-
name: `Narration ${tts.segmentIndex + 1}`,
|
|
949
|
-
url: tts.path,
|
|
950
|
-
type: "audio",
|
|
951
|
-
duration: tts.duration,
|
|
952
|
-
});
|
|
953
|
-
|
|
954
|
-
project.addClip({
|
|
955
|
-
sourceId: audioSource.id,
|
|
956
|
-
trackId: audioTrack.id,
|
|
957
|
-
startTime: segment.startTime,
|
|
958
|
-
duration: tts.duration,
|
|
959
|
-
sourceStartOffset: 0,
|
|
960
|
-
sourceEndOffset: tts.duration,
|
|
961
|
-
});
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
// Add video/image sources and clips
|
|
965
|
-
let currentTime = 0;
|
|
966
|
-
const videoClipIds: string[] = [];
|
|
967
|
-
const fadeDuration = 0.3; // Fade duration in seconds for smooth transitions
|
|
968
|
-
|
|
969
|
-
for (let i = 0; i < segments.length; i++) {
|
|
970
|
-
const segment = segments[i];
|
|
971
|
-
const hasVideo = videoPaths[i] && videoPaths[i] !== "";
|
|
972
|
-
const hasImage = imagePaths[i] && imagePaths[i] !== "";
|
|
973
|
-
|
|
974
|
-
if (!hasVideo && !hasImage) {
|
|
975
|
-
// Skip if no visual asset
|
|
976
|
-
currentTime += segment.duration;
|
|
977
|
-
continue;
|
|
978
|
-
}
|
|
979
|
-
|
|
980
|
-
const assetPath = hasVideo ? videoPaths[i] : imagePaths[i];
|
|
981
|
-
const mediaType = hasVideo ? "video" : "image";
|
|
982
|
-
|
|
983
|
-
const source = project.addSource({
|
|
984
|
-
name: `Scene ${i + 1}`,
|
|
985
|
-
url: assetPath,
|
|
986
|
-
type: mediaType as "video" | "image",
|
|
987
|
-
duration: segment.duration,
|
|
988
|
-
});
|
|
989
|
-
|
|
990
|
-
const clip = project.addClip({
|
|
991
|
-
sourceId: source.id,
|
|
992
|
-
trackId: videoTrack.id,
|
|
993
|
-
startTime: currentTime,
|
|
994
|
-
duration: segment.duration,
|
|
995
|
-
sourceStartOffset: 0,
|
|
996
|
-
sourceEndOffset: segment.duration,
|
|
997
|
-
});
|
|
998
|
-
|
|
999
|
-
videoClipIds.push(clip.id);
|
|
1000
|
-
currentTime += segment.duration;
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
// Add fade effects to video clips for smoother scene transitions
|
|
1004
|
-
for (let i = 0; i < videoClipIds.length; i++) {
|
|
1005
|
-
const clipId = videoClipIds[i];
|
|
1006
|
-
const clip = project.getClips().find(c => c.id === clipId);
|
|
1007
|
-
if (!clip) continue;
|
|
1008
|
-
|
|
1009
|
-
// Add fadeIn effect (except for first clip)
|
|
1010
|
-
if (i > 0) {
|
|
1011
|
-
project.addEffect(clipId, {
|
|
1012
|
-
type: "fadeIn",
|
|
1013
|
-
startTime: 0,
|
|
1014
|
-
duration: fadeDuration,
|
|
1015
|
-
params: {},
|
|
1016
|
-
});
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
// Add fadeOut effect (except for last clip)
|
|
1020
|
-
if (i < videoClipIds.length - 1) {
|
|
1021
|
-
project.addEffect(clipId, {
|
|
1022
|
-
type: "fadeOut",
|
|
1023
|
-
startTime: clip.duration - fadeDuration,
|
|
1024
|
-
duration: fadeDuration,
|
|
1025
|
-
params: {},
|
|
1026
|
-
});
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
// Save project file
|
|
1031
|
-
let outputPath = resolve(process.cwd(), options.output);
|
|
1032
|
-
|
|
1033
|
-
// Detect if output looks like a directory (ends with / or no .json extension)
|
|
1034
|
-
const looksLikeDirectory =
|
|
1035
|
-
options.output.endsWith("/") ||
|
|
1036
|
-
(!options.output.endsWith(".json") &&
|
|
1037
|
-
!options.output.endsWith(".vibe.json"));
|
|
1038
|
-
|
|
1039
|
-
if (looksLikeDirectory) {
|
|
1040
|
-
// Create directory if it doesn't exist
|
|
1041
|
-
if (!existsSync(outputPath)) {
|
|
1042
|
-
await mkdir(outputPath, { recursive: true });
|
|
1043
|
-
}
|
|
1044
|
-
outputPath = resolve(outputPath, "project.vibe.json");
|
|
1045
|
-
} else if (
|
|
1046
|
-
existsSync(outputPath) &&
|
|
1047
|
-
(await stat(outputPath)).isDirectory()
|
|
1048
|
-
) {
|
|
1049
|
-
// Existing directory without trailing slash
|
|
1050
|
-
outputPath = resolve(outputPath, "project.vibe.json");
|
|
1051
|
-
} else {
|
|
1052
|
-
// File path - ensure parent directory exists
|
|
1053
|
-
const parentDir = dirname(outputPath);
|
|
1054
|
-
if (!existsSync(parentDir)) {
|
|
1055
|
-
await mkdir(parentDir, { recursive: true });
|
|
1056
|
-
}
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
await writeFile(
|
|
1060
|
-
outputPath,
|
|
1061
|
-
JSON.stringify(project.toJSON(), null, 2),
|
|
1062
|
-
"utf-8"
|
|
1063
|
-
);
|
|
1064
|
-
|
|
1065
|
-
assembleSpinner.succeed(chalk.green("Project assembled"));
|
|
1066
|
-
|
|
1067
|
-
// Step 6: AI Review (optional)
|
|
1068
|
-
if (options.review) {
|
|
1069
|
-
const reviewSpinner = ora("Reviewing video with Gemini AI...").start();
|
|
1070
|
-
try {
|
|
1071
|
-
const reviewTarget = videoPaths.find((p) => p && p !== "");
|
|
1072
|
-
if (reviewTarget) {
|
|
1073
|
-
const storyboardFile = resolve(effectiveOutputDir, "storyboard.json");
|
|
1074
|
-
const reviewResult = await executeReview({
|
|
1075
|
-
videoPath: reviewTarget,
|
|
1076
|
-
storyboardPath: existsSync(storyboardFile) ? storyboardFile : undefined,
|
|
1077
|
-
autoApply: options.reviewAutoApply,
|
|
1078
|
-
model: "flash",
|
|
1079
|
-
});
|
|
1080
|
-
|
|
1081
|
-
if (reviewResult.success && reviewResult.feedback) {
|
|
1082
|
-
reviewSpinner.succeed(chalk.green(`AI Review: ${reviewResult.feedback.overallScore}/10`));
|
|
1083
|
-
if (reviewResult.appliedFixes && reviewResult.appliedFixes.length > 0) {
|
|
1084
|
-
for (const fix of reviewResult.appliedFixes) {
|
|
1085
|
-
console.log(chalk.green(` + ${fix}`));
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
if (reviewResult.feedback.recommendations.length > 0) {
|
|
1089
|
-
for (const rec of reviewResult.feedback.recommendations) {
|
|
1090
|
-
console.log(chalk.dim(` * ${rec}`));
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
} else {
|
|
1094
|
-
reviewSpinner.warn(chalk.yellow("AI review completed but no actionable feedback"));
|
|
1095
|
-
}
|
|
1096
|
-
} else {
|
|
1097
|
-
reviewSpinner.warn(chalk.yellow("No videos available for review"));
|
|
1098
|
-
}
|
|
1099
|
-
} catch {
|
|
1100
|
-
reviewSpinner.warn(chalk.yellow("AI review skipped (non-critical error)"));
|
|
1101
|
-
}
|
|
1102
|
-
console.log();
|
|
1103
|
-
}
|
|
1104
|
-
|
|
1105
|
-
// Final summary
|
|
1106
|
-
console.log();
|
|
1107
|
-
console.log(chalk.bold.green("Script-to-Video complete!"));
|
|
1108
|
-
console.log(chalk.dim("─".repeat(60)));
|
|
1109
|
-
console.log();
|
|
1110
|
-
console.log(` 📄 Project: ${chalk.cyan(outputPath)}`);
|
|
1111
|
-
console.log(` 🎬 Scenes: ${segments.length}`);
|
|
1112
|
-
console.log(` ⏱️ Duration: ${totalDuration}s`);
|
|
1113
|
-
console.log(` 📁 Assets: ${effectiveOutputDir}/`);
|
|
1114
|
-
if (perSceneTTS.length > 0 || failedNarrations.length > 0) {
|
|
1115
|
-
const narrationInfo = `${perSceneTTS.length}/${segments.length}`;
|
|
1116
|
-
if (failedNarrations.length > 0) {
|
|
1117
|
-
const failedSceneNums = failedNarrations.map((f) => f.sceneNum).join(", ");
|
|
1118
|
-
console.log(` 🎙️ Narrations: ${narrationInfo} narration-*.mp3`);
|
|
1119
|
-
console.log(chalk.yellow(` ⚠ Failed: scene ${failedSceneNums}`));
|
|
1120
|
-
} else {
|
|
1121
|
-
console.log(` 🎙️ Narrations: ${perSceneTTS.length} narration-*.mp3`);
|
|
1122
|
-
}
|
|
1123
|
-
}
|
|
1124
|
-
console.log(` 🖼️ Images: ${successfulImages} scene-*.png`);
|
|
1125
|
-
if (!options.imagesOnly) {
|
|
1126
|
-
const videoCount = videoPaths.filter((p) => p && p !== "").length;
|
|
1127
|
-
console.log(` 🎥 Videos: ${videoCount}/${segments.length} scene-*.mp4`);
|
|
1128
|
-
if (failedScenes.length > 0) {
|
|
1129
|
-
const uniqueFailedScenes = [...new Set(failedScenes)].sort((a, b) => a - b);
|
|
1130
|
-
console.log(chalk.yellow(` ⚠ Failed: scene ${uniqueFailedScenes.join(", ")} (fallback to image)`));
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
console.log();
|
|
1134
|
-
console.log(chalk.dim("Next steps:"));
|
|
1135
|
-
console.log(chalk.dim(` vibe project info ${options.output}`));
|
|
1136
|
-
console.log(chalk.dim(` vibe export ${options.output} -o final.mp4`));
|
|
1137
|
-
|
|
1138
|
-
// Show regeneration hint if there were failures
|
|
1139
|
-
if (!options.imagesOnly && failedScenes.length > 0) {
|
|
1140
|
-
const uniqueFailedScenes = [...new Set(failedScenes)].sort((a, b) => a - b);
|
|
1141
|
-
console.log();
|
|
1142
|
-
console.log(chalk.dim("💡 To regenerate failed scenes:"));
|
|
1143
|
-
for (const sceneNum of uniqueFailedScenes) {
|
|
1144
|
-
console.log(chalk.dim(` vibe ai regenerate-scene ${effectiveOutputDir}/ --scene ${sceneNum} --video-only`));
|
|
1145
|
-
}
|
|
1146
|
-
}
|
|
1147
|
-
console.log();
|
|
1148
|
-
} catch (error) {
|
|
1149
|
-
console.error(chalk.red("Script-to-Video failed"));
|
|
1150
|
-
console.error(error);
|
|
1151
|
-
process.exit(1);
|
|
1152
|
-
}
|
|
1153
|
-
});
|
|
1154
|
-
|
|
1155
|
-
// Regenerate Scene command
|
|
1156
|
-
aiCommand
|
|
1157
|
-
.command("regenerate-scene")
|
|
1158
|
-
.description("Regenerate a specific scene in a script-to-video project")
|
|
1159
|
-
.argument("<project-dir>", "Path to the script-to-video output directory")
|
|
1160
|
-
.requiredOption("--scene <numbers>", "Scene number(s) to regenerate (1-based), e.g., 3 or 3,4,5")
|
|
1161
|
-
.option("--video-only", "Only regenerate video")
|
|
1162
|
-
.option("--narration-only", "Only regenerate narration")
|
|
1163
|
-
.option("--image-only", "Only regenerate image")
|
|
1164
|
-
.option("-g, --generator <engine>", "Video generator: kling | runway | veo", "kling")
|
|
1165
|
-
.option("-i, --image-provider <provider>", "Image provider: gemini | openai | grok", "gemini")
|
|
1166
|
-
.option("-v, --voice <id>", "ElevenLabs voice ID for narration")
|
|
1167
|
-
.option("-a, --aspect-ratio <ratio>", "Aspect ratio: 16:9 | 9:16 | 1:1", "16:9")
|
|
1168
|
-
.option("--retries <count>", "Number of retries for video generation failures", String(DEFAULT_VIDEO_RETRIES))
|
|
1169
|
-
.option("--reference-scene <num>", "Use another scene's image as reference for character consistency")
|
|
1170
|
-
.action(async (projectDir: string, options) => {
|
|
1171
|
-
try {
|
|
1172
|
-
const outputDir = resolve(process.cwd(), projectDir);
|
|
1173
|
-
const storyboardPath = resolve(outputDir, "storyboard.json");
|
|
1174
|
-
const projectPath = resolve(outputDir, "project.vibe.json");
|
|
1175
|
-
|
|
1176
|
-
// Validate project directory
|
|
1177
|
-
if (!existsSync(outputDir)) {
|
|
1178
|
-
console.error(chalk.red(`Project directory not found: ${outputDir}`));
|
|
1179
|
-
process.exit(1);
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
if (!existsSync(storyboardPath)) {
|
|
1183
|
-
console.error(chalk.red(`Storyboard not found: ${storyboardPath}`));
|
|
1184
|
-
console.error(chalk.dim("This command requires a storyboard.json file from script-to-video output"));
|
|
1185
|
-
process.exit(1);
|
|
1186
|
-
}
|
|
1187
|
-
|
|
1188
|
-
// Parse scene number(s) - supports "3" or "3,4,5"
|
|
1189
|
-
const sceneNums = options.scene.split(",").map((s: string) => parseInt(s.trim())).filter((n: number) => !isNaN(n) && n >= 1);
|
|
1190
|
-
if (sceneNums.length === 0) {
|
|
1191
|
-
console.error(chalk.red("Scene number must be a positive integer (1-based), e.g., --scene 3 or --scene 3,4,5"));
|
|
1192
|
-
process.exit(1);
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
// Load storyboard
|
|
1196
|
-
const storyboardContent = await readFile(storyboardPath, "utf-8");
|
|
1197
|
-
const segments: StoryboardSegment[] = JSON.parse(storyboardContent);
|
|
1198
|
-
|
|
1199
|
-
// Validate all scene numbers
|
|
1200
|
-
for (const sceneNum of sceneNums) {
|
|
1201
|
-
if (sceneNum > segments.length) {
|
|
1202
|
-
console.error(chalk.red(`Scene ${sceneNum} does not exist. Storyboard has ${segments.length} scenes.`));
|
|
1203
|
-
process.exit(1);
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
// Determine what to regenerate
|
|
1208
|
-
const regenerateVideo = options.videoOnly || (!options.narrationOnly && !options.imageOnly);
|
|
1209
|
-
const regenerateNarration = options.narrationOnly || (!options.videoOnly && !options.imageOnly);
|
|
1210
|
-
const regenerateImage = options.imageOnly || (!options.videoOnly && !options.narrationOnly);
|
|
1211
|
-
|
|
1212
|
-
console.log();
|
|
1213
|
-
console.log(chalk.bold.cyan(`🔄 Regenerating Scene${sceneNums.length > 1 ? "s" : ""} ${sceneNums.join(", ")}`));
|
|
1214
|
-
console.log(chalk.dim("─".repeat(60)));
|
|
1215
|
-
console.log();
|
|
1216
|
-
console.log(` 📁 Project: ${outputDir}`);
|
|
1217
|
-
console.log(` 🎬 Scenes: ${sceneNums.join(", ")} of ${segments.length}`);
|
|
1218
|
-
console.log();
|
|
1219
|
-
|
|
1220
|
-
// Get required API keys (once, before processing scenes)
|
|
1221
|
-
let imageApiKey: string | undefined;
|
|
1222
|
-
let videoApiKey: string | undefined;
|
|
1223
|
-
let elevenlabsApiKey: string | undefined;
|
|
1224
|
-
|
|
1225
|
-
if (regenerateImage) {
|
|
1226
|
-
const imageProvider = options.imageProvider || "openai";
|
|
1227
|
-
if (imageProvider === "openai" || imageProvider === "dalle") {
|
|
1228
|
-
imageApiKey = (await getApiKey("OPENAI_API_KEY", "OpenAI")) ?? undefined;
|
|
1229
|
-
if (!imageApiKey) {
|
|
1230
|
-
console.error(chalk.red("OpenAI API key required for image generation. Set OPENAI_API_KEY in .env or run: vibe setup"));
|
|
1231
|
-
process.exit(1);
|
|
1232
|
-
}
|
|
1233
|
-
} else if (imageProvider === "gemini") {
|
|
1234
|
-
imageApiKey = (await getApiKey("GOOGLE_API_KEY", "Google")) ?? undefined;
|
|
1235
|
-
if (!imageApiKey) {
|
|
1236
|
-
console.error(chalk.red("Google API key required for Gemini image generation. Set GOOGLE_API_KEY in .env or run: vibe setup"));
|
|
1237
|
-
process.exit(1);
|
|
1238
|
-
}
|
|
1239
|
-
} else if (imageProvider === "grok") {
|
|
1240
|
-
imageApiKey = (await getApiKey("XAI_API_KEY", "xAI")) ?? undefined;
|
|
1241
|
-
if (!imageApiKey) {
|
|
1242
|
-
console.error(chalk.red("xAI API key required for Grok image generation. Set XAI_API_KEY in .env or run: vibe setup"));
|
|
1243
|
-
process.exit(1);
|
|
1244
|
-
}
|
|
1245
|
-
}
|
|
1246
|
-
}
|
|
1247
|
-
|
|
1248
|
-
if (regenerateVideo) {
|
|
1249
|
-
if (options.generator === "kling") {
|
|
1250
|
-
const key = await getApiKey("KLING_API_KEY", "Kling");
|
|
1251
|
-
if (!key) {
|
|
1252
|
-
console.error(chalk.red("Kling API key required. Set KLING_API_KEY in .env or run: vibe setup"));
|
|
1253
|
-
process.exit(1);
|
|
1254
|
-
}
|
|
1255
|
-
videoApiKey = key;
|
|
1256
|
-
} else {
|
|
1257
|
-
const key = await getApiKey("RUNWAY_API_SECRET", "Runway");
|
|
1258
|
-
if (!key) {
|
|
1259
|
-
console.error(chalk.red("Runway API key required. Set RUNWAY_API_SECRET in .env or run: vibe setup"));
|
|
1260
|
-
process.exit(1);
|
|
1261
|
-
}
|
|
1262
|
-
videoApiKey = key;
|
|
1263
|
-
}
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
if (regenerateNarration) {
|
|
1267
|
-
const key = await getApiKey("ELEVENLABS_API_KEY", "ElevenLabs");
|
|
1268
|
-
if (!key) {
|
|
1269
|
-
console.error(chalk.red("ElevenLabs API key required for narration. Set ELEVENLABS_API_KEY in .env or run: vibe setup"));
|
|
1270
|
-
process.exit(1);
|
|
1271
|
-
}
|
|
1272
|
-
elevenlabsApiKey = key;
|
|
1273
|
-
}
|
|
1274
|
-
|
|
1275
|
-
// Process each scene
|
|
1276
|
-
for (const sceneNum of sceneNums) {
|
|
1277
|
-
const segment = segments[sceneNum - 1];
|
|
1278
|
-
|
|
1279
|
-
console.log(chalk.cyan(`\n── Scene ${sceneNum} ──`));
|
|
1280
|
-
console.log(chalk.dim(` ${segment.description.slice(0, 50)}...`));
|
|
1281
|
-
|
|
1282
|
-
// Step 1: Regenerate narration if needed
|
|
1283
|
-
const narrationPath = resolve(outputDir, `narration-${sceneNum}.mp3`);
|
|
1284
|
-
let narrationDuration = segment.duration;
|
|
1285
|
-
|
|
1286
|
-
if (regenerateNarration && elevenlabsApiKey) {
|
|
1287
|
-
const ttsSpinner = ora(`🎙️ Regenerating narration for scene ${sceneNum}...`).start();
|
|
1288
|
-
|
|
1289
|
-
const elevenlabs = new ElevenLabsProvider();
|
|
1290
|
-
await elevenlabs.initialize({ apiKey: elevenlabsApiKey });
|
|
1291
|
-
|
|
1292
|
-
const narrationText = segment.narration || segment.description;
|
|
1293
|
-
|
|
1294
|
-
const ttsResult = await elevenlabs.textToSpeech(narrationText, {
|
|
1295
|
-
voiceId: options.voice,
|
|
1296
|
-
});
|
|
1297
|
-
|
|
1298
|
-
if (!ttsResult.success || !ttsResult.audioBuffer) {
|
|
1299
|
-
ttsSpinner.fail(chalk.red(`Failed to generate narration: ${ttsResult.error || "Unknown error"}`));
|
|
1300
|
-
process.exit(1);
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
await writeFile(narrationPath, ttsResult.audioBuffer);
|
|
1304
|
-
narrationDuration = await getAudioDuration(narrationPath);
|
|
1305
|
-
|
|
1306
|
-
// Update segment duration in storyboard
|
|
1307
|
-
segment.duration = narrationDuration;
|
|
1308
|
-
|
|
1309
|
-
ttsSpinner.succeed(chalk.green(`Generated narration (${narrationDuration.toFixed(1)}s)`));
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
// Step 2: Regenerate image if needed
|
|
1313
|
-
const imagePath = resolve(outputDir, `scene-${sceneNum}.png`);
|
|
1314
|
-
|
|
1315
|
-
if (regenerateImage && imageApiKey) {
|
|
1316
|
-
const imageSpinner = ora(`🎨 Regenerating image for scene ${sceneNum}...`).start();
|
|
1317
|
-
|
|
1318
|
-
const imageProvider = options.imageProvider || "gemini";
|
|
1319
|
-
|
|
1320
|
-
// Build prompt with character description for consistency
|
|
1321
|
-
const characterDesc = segment.characterDescription || segments[0]?.characterDescription;
|
|
1322
|
-
let imagePrompt = segment.visualStyle
|
|
1323
|
-
? `${segment.visuals}. Style: ${segment.visualStyle}`
|
|
1324
|
-
: segment.visuals;
|
|
1325
|
-
|
|
1326
|
-
// Add character description to prompt if available
|
|
1327
|
-
if (characterDesc) {
|
|
1328
|
-
imagePrompt = `${imagePrompt}\n\nIMPORTANT - Character appearance must match exactly: ${characterDesc}`;
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
// Check if we should use reference-based generation for character consistency
|
|
1332
|
-
const refSceneNum = options.referenceScene ? parseInt(options.referenceScene) : null;
|
|
1333
|
-
let referenceImageBuffer: Buffer | undefined;
|
|
1334
|
-
|
|
1335
|
-
if (refSceneNum && refSceneNum >= 1 && refSceneNum <= segments.length && refSceneNum !== sceneNum) {
|
|
1336
|
-
const refImagePath = resolve(outputDir, `scene-${refSceneNum}.png`);
|
|
1337
|
-
if (existsSync(refImagePath)) {
|
|
1338
|
-
referenceImageBuffer = await readFile(refImagePath);
|
|
1339
|
-
imageSpinner.text = `🎨 Regenerating image for scene ${sceneNum} (using scene ${refSceneNum} as reference)...`;
|
|
1340
|
-
}
|
|
1341
|
-
} else if (!refSceneNum) {
|
|
1342
|
-
// Auto-detect: use the first available scene image as reference
|
|
1343
|
-
for (let i = 1; i <= segments.length; i++) {
|
|
1344
|
-
if (i !== sceneNum) {
|
|
1345
|
-
const otherImagePath = resolve(outputDir, `scene-${i}.png`);
|
|
1346
|
-
if (existsSync(otherImagePath)) {
|
|
1347
|
-
referenceImageBuffer = await readFile(otherImagePath);
|
|
1348
|
-
imageSpinner.text = `🎨 Regenerating image for scene ${sceneNum} (using scene ${i} as reference)...`;
|
|
1349
|
-
break;
|
|
1350
|
-
}
|
|
1351
|
-
}
|
|
1352
|
-
}
|
|
1353
|
-
}
|
|
1354
|
-
|
|
1355
|
-
// Determine image size/aspect ratio based on provider
|
|
1356
|
-
const dalleImageSizes: Record<string, "1536x1024" | "1024x1536" | "1024x1024"> = {
|
|
1357
|
-
"16:9": "1536x1024",
|
|
1358
|
-
"9:16": "1024x1536",
|
|
1359
|
-
"1:1": "1024x1024",
|
|
1360
|
-
};
|
|
1361
|
-
let imageBuffer: Buffer | undefined;
|
|
1362
|
-
let imageUrl: string | undefined;
|
|
1363
|
-
let imageError: string | undefined;
|
|
1364
|
-
|
|
1365
|
-
if (imageProvider === "openai" || imageProvider === "dalle") {
|
|
1366
|
-
const openaiImage = new OpenAIImageProvider();
|
|
1367
|
-
await openaiImage.initialize({ apiKey: imageApiKey });
|
|
1368
|
-
const imageResult = await openaiImage.generateImage(imagePrompt, {
|
|
1369
|
-
size: dalleImageSizes[options.aspectRatio] || "1536x1024",
|
|
1370
|
-
quality: "standard",
|
|
1371
|
-
});
|
|
1372
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
1373
|
-
imageUrl = imageResult.images[0].url;
|
|
1374
|
-
} else {
|
|
1375
|
-
imageError = imageResult.error;
|
|
1376
|
-
}
|
|
1377
|
-
} else if (imageProvider === "gemini") {
|
|
1378
|
-
const gemini = new GeminiProvider();
|
|
1379
|
-
await gemini.initialize({ apiKey: imageApiKey });
|
|
1380
|
-
|
|
1381
|
-
// Use editImage with reference for character consistency
|
|
1382
|
-
if (referenceImageBuffer) {
|
|
1383
|
-
// Extract the main action from the scene description (take first action if multiple)
|
|
1384
|
-
const simplifiedVisuals = segment.visuals.split(/[,.]/).find((part: string) =>
|
|
1385
|
-
part.includes("standing") || part.includes("sitting") || part.includes("walking") ||
|
|
1386
|
-
part.includes("lying") || part.includes("reaching") || part.includes("looking") ||
|
|
1387
|
-
part.includes("working") || part.includes("coding") || part.includes("typing")
|
|
1388
|
-
) || segment.visuals.split(".")[0];
|
|
1389
|
-
|
|
1390
|
-
const editPrompt = `Generate a new image showing the SAME SINGLE person from the reference image in a new scene.
|
|
1391
|
-
|
|
1392
|
-
REFERENCE: Look at the person in the reference image - their face, hair, build, and overall appearance.
|
|
1393
|
-
|
|
1394
|
-
NEW SCENE: ${simplifiedVisuals}
|
|
1395
|
-
|
|
1396
|
-
CRITICAL RULES:
|
|
1397
|
-
1. Show ONLY ONE person - the exact same individual from the reference image
|
|
1398
|
-
2. The person must have the IDENTICAL face, hair style, and body type
|
|
1399
|
-
3. Do NOT show multiple people or duplicate the character
|
|
1400
|
-
4. Create a single moment in time, one pose, one action
|
|
1401
|
-
5. Match the art style and quality of the reference image
|
|
1402
|
-
|
|
1403
|
-
Generate the single-person scene image now.`;
|
|
1404
|
-
|
|
1405
|
-
const imageResult = await gemini.editImage([referenceImageBuffer], editPrompt, {
|
|
1406
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
1407
|
-
});
|
|
1408
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
1409
|
-
const img = imageResult.images[0];
|
|
1410
|
-
if (img.base64) {
|
|
1411
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
1412
|
-
}
|
|
1413
|
-
} else {
|
|
1414
|
-
imageError = imageResult.error;
|
|
1415
|
-
}
|
|
1416
|
-
} else {
|
|
1417
|
-
// No reference image, use regular generation
|
|
1418
|
-
const imageResult = await gemini.generateImage(imagePrompt, {
|
|
1419
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
1420
|
-
});
|
|
1421
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
1422
|
-
const img = imageResult.images[0];
|
|
1423
|
-
if (img.base64) {
|
|
1424
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
1425
|
-
}
|
|
1426
|
-
} else {
|
|
1427
|
-
imageError = imageResult.error;
|
|
1428
|
-
}
|
|
1429
|
-
}
|
|
1430
|
-
} else if (imageProvider === "grok") {
|
|
1431
|
-
const { GrokProvider } = await import("@vibeframe/ai-providers");
|
|
1432
|
-
const grok = new GrokProvider();
|
|
1433
|
-
await grok.initialize({ apiKey: imageApiKey });
|
|
1434
|
-
const imageResult = await grok.generateImage(imagePrompt, {
|
|
1435
|
-
aspectRatio: options.aspectRatio || "16:9",
|
|
1436
|
-
});
|
|
1437
|
-
if (imageResult.success && imageResult.images && imageResult.images.length > 0) {
|
|
1438
|
-
const img = imageResult.images[0];
|
|
1439
|
-
if (img.base64) {
|
|
1440
|
-
imageBuffer = Buffer.from(img.base64, "base64");
|
|
1441
|
-
} else if (img.url) {
|
|
1442
|
-
imageUrl = img.url;
|
|
1443
|
-
}
|
|
1444
|
-
} else {
|
|
1445
|
-
imageError = imageResult.error;
|
|
1446
|
-
}
|
|
1447
|
-
}
|
|
1448
|
-
|
|
1449
|
-
if (imageBuffer) {
|
|
1450
|
-
await writeFile(imagePath, imageBuffer);
|
|
1451
|
-
imageSpinner.succeed(chalk.green("Generated image"));
|
|
1452
|
-
} else if (imageUrl) {
|
|
1453
|
-
const response = await fetch(imageUrl);
|
|
1454
|
-
const buffer = Buffer.from(await response.arrayBuffer());
|
|
1455
|
-
await writeFile(imagePath, buffer);
|
|
1456
|
-
imageSpinner.succeed(chalk.green("Generated image"));
|
|
1457
|
-
} else {
|
|
1458
|
-
const errorMsg = imageError || "Unknown error";
|
|
1459
|
-
imageSpinner.fail(chalk.red(`Failed to generate image: ${errorMsg}`));
|
|
1460
|
-
process.exit(1);
|
|
1461
|
-
}
|
|
1462
|
-
}
|
|
1463
|
-
|
|
1464
|
-
// Step 3: Regenerate video if needed
|
|
1465
|
-
const videoPath = resolve(outputDir, `scene-${sceneNum}.mp4`);
|
|
1466
|
-
|
|
1467
|
-
if (regenerateVideo && videoApiKey) {
|
|
1468
|
-
const videoSpinner = ora(`🎬 Regenerating video for scene ${sceneNum}...`).start();
|
|
1469
|
-
|
|
1470
|
-
// Check if image exists
|
|
1471
|
-
if (!existsSync(imagePath)) {
|
|
1472
|
-
videoSpinner.fail(chalk.red(`Reference image not found: ${imagePath}`));
|
|
1473
|
-
console.error(chalk.dim("Generate an image first with --image-only or regenerate all assets"));
|
|
1474
|
-
process.exit(1);
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
const imageBuffer = await readFile(imagePath);
|
|
1478
|
-
const ext = extname(imagePath).toLowerCase().slice(1);
|
|
1479
|
-
const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
|
|
1480
|
-
const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
|
|
1481
|
-
|
|
1482
|
-
const videoDuration = (segment.duration > 5 ? 10 : 5) as 5 | 10;
|
|
1483
|
-
const maxRetries = parseInt(options.retries) || DEFAULT_VIDEO_RETRIES;
|
|
1484
|
-
|
|
1485
|
-
let videoGenerated = false;
|
|
1486
|
-
|
|
1487
|
-
if (options.generator === "kling") {
|
|
1488
|
-
const kling = new KlingProvider();
|
|
1489
|
-
await kling.initialize({ apiKey: videoApiKey });
|
|
1490
|
-
|
|
1491
|
-
if (!kling.isConfigured()) {
|
|
1492
|
-
videoSpinner.fail(chalk.red("Invalid Kling API key format. Use ACCESS_KEY:SECRET_KEY"));
|
|
1493
|
-
process.exit(1);
|
|
1494
|
-
}
|
|
1495
|
-
|
|
1496
|
-
// Try to use image-to-video if ImgBB API key is available
|
|
1497
|
-
const imgbbApiKey = await getApiKeyFromConfig("imgbb") || process.env.IMGBB_API_KEY;
|
|
1498
|
-
let imageUrl: string | undefined;
|
|
1499
|
-
|
|
1500
|
-
if (imgbbApiKey) {
|
|
1501
|
-
videoSpinner.text = `🎬 Uploading image to ImgBB...`;
|
|
1502
|
-
const uploadResult = await uploadToImgbb(imageBuffer, imgbbApiKey);
|
|
1503
|
-
if (uploadResult.success && uploadResult.url) {
|
|
1504
|
-
imageUrl = uploadResult.url;
|
|
1505
|
-
videoSpinner.text = `🎬 Starting image-to-video generation...`;
|
|
1506
|
-
} else {
|
|
1507
|
-
console.log(chalk.yellow(`\n ⚠ ImgBB upload failed, falling back to text-to-video`));
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
|
|
1511
|
-
const result = await generateVideoWithRetryKling(
|
|
1512
|
-
kling,
|
|
1513
|
-
segment,
|
|
1514
|
-
{
|
|
1515
|
-
duration: videoDuration,
|
|
1516
|
-
aspectRatio: options.aspectRatio as "16:9" | "9:16" | "1:1",
|
|
1517
|
-
referenceImage: imageUrl, // Use uploaded URL for image-to-video
|
|
1518
|
-
},
|
|
1519
|
-
maxRetries
|
|
1520
|
-
);
|
|
1521
|
-
|
|
1522
|
-
if (result) {
|
|
1523
|
-
videoSpinner.text = `🎬 Waiting for video to complete...`;
|
|
1524
|
-
|
|
1525
|
-
for (let attempt = 0; attempt <= maxRetries && !videoGenerated; attempt++) {
|
|
1526
|
-
try {
|
|
1527
|
-
const waitResult = await kling.waitForCompletion(
|
|
1528
|
-
result.taskId,
|
|
1529
|
-
result.type,
|
|
1530
|
-
(status) => {
|
|
1531
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: ${status.status}...`;
|
|
1532
|
-
},
|
|
1533
|
-
600000
|
|
1534
|
-
);
|
|
1535
|
-
|
|
1536
|
-
if (waitResult.status === "completed" && waitResult.videoUrl) {
|
|
1537
|
-
const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
|
|
1538
|
-
await writeFile(videoPath, buffer);
|
|
1539
|
-
|
|
1540
|
-
// Extend video to match narration duration if needed
|
|
1541
|
-
await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${sceneNum}`, {
|
|
1542
|
-
kling,
|
|
1543
|
-
videoId: waitResult.videoId,
|
|
1544
|
-
onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
|
|
1545
|
-
});
|
|
1546
|
-
|
|
1547
|
-
videoGenerated = true;
|
|
1548
|
-
} else if (attempt < maxRetries) {
|
|
1549
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: Retry ${attempt + 1}/${maxRetries}...`;
|
|
1550
|
-
await sleep(RETRY_DELAY_MS);
|
|
1551
|
-
}
|
|
1552
|
-
} catch (err) {
|
|
1553
|
-
if (attempt >= maxRetries) {
|
|
1554
|
-
throw err;
|
|
1555
|
-
}
|
|
1556
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: Error, retry ${attempt + 1}/${maxRetries}...`;
|
|
1557
|
-
await sleep(RETRY_DELAY_MS);
|
|
1558
|
-
}
|
|
1559
|
-
}
|
|
1560
|
-
}
|
|
1561
|
-
} else {
|
|
1562
|
-
// Runway
|
|
1563
|
-
const runway = new RunwayProvider();
|
|
1564
|
-
await runway.initialize({ apiKey: videoApiKey });
|
|
1565
|
-
|
|
1566
|
-
const result = await generateVideoWithRetryRunway(
|
|
1567
|
-
runway,
|
|
1568
|
-
segment,
|
|
1569
|
-
referenceImage,
|
|
1570
|
-
{
|
|
1571
|
-
duration: videoDuration,
|
|
1572
|
-
aspectRatio: options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio as "16:9" | "9:16"),
|
|
1573
|
-
},
|
|
1574
|
-
maxRetries,
|
|
1575
|
-
(msg) => {
|
|
1576
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: ${msg}`;
|
|
1577
|
-
}
|
|
1578
|
-
);
|
|
1579
|
-
|
|
1580
|
-
if (result) {
|
|
1581
|
-
videoSpinner.text = `🎬 Waiting for video to complete...`;
|
|
1582
|
-
|
|
1583
|
-
for (let attempt = 0; attempt <= maxRetries && !videoGenerated; attempt++) {
|
|
1584
|
-
try {
|
|
1585
|
-
const waitResult = await runway.waitForCompletion(
|
|
1586
|
-
result.taskId,
|
|
1587
|
-
(status) => {
|
|
1588
|
-
const progress = status.progress !== undefined ? `${status.progress}%` : status.status;
|
|
1589
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: ${progress}...`;
|
|
1590
|
-
},
|
|
1591
|
-
300000
|
|
1592
|
-
);
|
|
1593
|
-
|
|
1594
|
-
if (waitResult.status === "completed" && waitResult.videoUrl) {
|
|
1595
|
-
const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
|
|
1596
|
-
await writeFile(videoPath, buffer);
|
|
1597
|
-
|
|
1598
|
-
// Extend video to match narration duration if needed (Runway - no Kling extend)
|
|
1599
|
-
await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${sceneNum}`, {
|
|
1600
|
-
onProgress: (msg) => { videoSpinner.text = `🎬 ${msg}`; },
|
|
1601
|
-
});
|
|
1602
|
-
|
|
1603
|
-
videoGenerated = true;
|
|
1604
|
-
} else if (attempt < maxRetries) {
|
|
1605
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: Retry ${attempt + 1}/${maxRetries}...`;
|
|
1606
|
-
await sleep(RETRY_DELAY_MS);
|
|
1607
|
-
}
|
|
1608
|
-
} catch (err) {
|
|
1609
|
-
if (attempt >= maxRetries) {
|
|
1610
|
-
throw err;
|
|
1611
|
-
}
|
|
1612
|
-
videoSpinner.text = `🎬 Scene ${sceneNum}: Error, retry ${attempt + 1}/${maxRetries}...`;
|
|
1613
|
-
await sleep(RETRY_DELAY_MS);
|
|
1614
|
-
}
|
|
1615
|
-
}
|
|
1616
|
-
}
|
|
1617
|
-
}
|
|
1618
|
-
|
|
1619
|
-
if (videoGenerated) {
|
|
1620
|
-
videoSpinner.succeed(chalk.green("Generated video"));
|
|
1621
|
-
} else {
|
|
1622
|
-
videoSpinner.fail(chalk.red("Failed to generate video after all retries"));
|
|
1623
|
-
process.exit(1);
|
|
1624
|
-
}
|
|
1625
|
-
}
|
|
1626
|
-
|
|
1627
|
-
// Step 4: Recalculate startTime for ALL segments and re-save storyboard
|
|
1628
|
-
{
|
|
1629
|
-
let currentTime = 0;
|
|
1630
|
-
for (const seg of segments) {
|
|
1631
|
-
seg.startTime = currentTime;
|
|
1632
|
-
currentTime += seg.duration;
|
|
1633
|
-
}
|
|
1634
|
-
await writeFile(storyboardPath, JSON.stringify(segments, null, 2), "utf-8");
|
|
1635
|
-
console.log(chalk.dim(` → Updated storyboard: ${storyboardPath}`));
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
|
-
// Step 5: Update project.vibe.json if it exists — update ALL clips' startTime/duration
|
|
1639
|
-
if (existsSync(projectPath)) {
|
|
1640
|
-
const updateSpinner = ora("📦 Updating project file...").start();
|
|
1641
|
-
|
|
1642
|
-
try {
|
|
1643
|
-
const projectContent = await readFile(projectPath, "utf-8");
|
|
1644
|
-
const projectData = JSON.parse(projectContent) as ProjectFile;
|
|
1645
|
-
|
|
1646
|
-
// Find and update the source for this scene
|
|
1647
|
-
const sceneName = `Scene ${sceneNum}`;
|
|
1648
|
-
const narrationName = `Narration ${sceneNum}`;
|
|
1649
|
-
|
|
1650
|
-
// Update video/image source
|
|
1651
|
-
const videoSource = projectData.state.sources.find((s) => s.name === sceneName);
|
|
1652
|
-
if (videoSource) {
|
|
1653
|
-
const hasVideo = existsSync(videoPath);
|
|
1654
|
-
videoSource.url = hasVideo ? videoPath : imagePath;
|
|
1655
|
-
videoSource.type = hasVideo ? "video" : "image";
|
|
1656
|
-
videoSource.duration = segment.duration;
|
|
1657
|
-
}
|
|
1658
|
-
|
|
1659
|
-
// Update narration source
|
|
1660
|
-
const narrationSource = projectData.state.sources.find((s) => s.name === narrationName);
|
|
1661
|
-
if (narrationSource && regenerateNarration) {
|
|
1662
|
-
narrationSource.duration = narrationDuration;
|
|
1663
|
-
}
|
|
1664
|
-
|
|
1665
|
-
// Update ALL clips' startTime and duration based on recalculated segments
|
|
1666
|
-
for (const clip of projectData.state.clips) {
|
|
1667
|
-
const source = projectData.state.sources.find((s) => s.id === clip.sourceId);
|
|
1668
|
-
if (!source) continue;
|
|
1669
|
-
|
|
1670
|
-
// Match source name to segment (e.g., "Scene 1" → segment 0, "Narration 2" → segment 1)
|
|
1671
|
-
const sceneMatch = source.name.match(/^Scene (\d+)$/);
|
|
1672
|
-
const narrationMatch = source.name.match(/^Narration (\d+)$/);
|
|
1673
|
-
const segIdx = sceneMatch ? parseInt(sceneMatch[1]) - 1 : narrationMatch ? parseInt(narrationMatch[1]) - 1 : -1;
|
|
1674
|
-
|
|
1675
|
-
if (segIdx >= 0 && segIdx < segments.length) {
|
|
1676
|
-
const seg = segments[segIdx];
|
|
1677
|
-
clip.startTime = seg.startTime;
|
|
1678
|
-
clip.duration = seg.duration;
|
|
1679
|
-
clip.sourceEndOffset = seg.duration;
|
|
1680
|
-
// Also update the source duration to match segment
|
|
1681
|
-
source.duration = seg.duration;
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
await writeFile(projectPath, JSON.stringify(projectData, null, 2), "utf-8");
|
|
1686
|
-
updateSpinner.succeed(chalk.green("Updated project file (all clips synced)"));
|
|
1687
|
-
} catch (err) {
|
|
1688
|
-
updateSpinner.warn(chalk.yellow(`Could not update project file: ${err}`));
|
|
1689
|
-
}
|
|
1690
|
-
}
|
|
1691
|
-
|
|
1692
|
-
console.log(chalk.green(` ✓ Scene ${sceneNum} done`));
|
|
1693
|
-
} // End of for loop over sceneNums
|
|
1694
|
-
|
|
1695
|
-
// Final summary
|
|
1696
|
-
console.log();
|
|
1697
|
-
console.log(chalk.bold.green(`✅ ${sceneNums.length} scene${sceneNums.length > 1 ? "s" : ""} regenerated successfully!`));
|
|
1698
|
-
console.log(chalk.dim("─".repeat(60)));
|
|
1699
|
-
console.log();
|
|
1700
|
-
console.log(chalk.dim("Next steps:"));
|
|
1701
|
-
console.log(chalk.dim(` vibe export ${outputDir}/ -o final.mp4`));
|
|
1702
|
-
console.log();
|
|
1703
|
-
} catch (error) {
|
|
1704
|
-
console.error(chalk.red("Scene regeneration failed"));
|
|
1705
|
-
console.error(error);
|
|
1706
|
-
process.exit(1);
|
|
1707
|
-
}
|
|
1708
|
-
});
|
|
1709
|
-
|
|
1710
|
-
}
|