@vibeframe/cli 0.48.6 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/tools/timeline.d.ts.map +1 -1
- package/dist/agent/tools/timeline.js +4 -2
- package/dist/agent/tools/timeline.js.map +1 -1
- package/dist/commands/ai-script-pipeline-cli.d.ts.map +1 -1
- package/dist/commands/ai-script-pipeline-cli.js +208 -1366
- package/dist/commands/ai-script-pipeline-cli.js.map +1 -1
- package/dist/commands/ai-script-pipeline.d.ts +19 -1
- package/dist/commands/ai-script-pipeline.d.ts.map +1 -1
- package/dist/commands/ai-script-pipeline.js +344 -39
- package/dist/commands/ai-script-pipeline.js.map +1 -1
- package/dist/commands/output.d.ts.map +1 -1
- package/dist/commands/output.js +9 -3
- package/dist/commands/output.js.map +1 -1
- package/dist/commands/output.test.js +12 -0
- package/dist/commands/output.test.js.map +1 -1
- package/dist/commands/timeline.js +3 -1
- package/dist/commands/timeline.js.map +1 -1
- package/dist/commands/timeline.test.js +14 -0
- package/dist/commands/timeline.test.js.map +1 -1
- package/dist/pipeline/renderers/__tests__/html-template.test.js +46 -0
- package/dist/pipeline/renderers/__tests__/html-template.test.js.map +1 -1
- package/dist/pipeline/renderers/__tests__/integration.test.js +64 -0
- package/dist/pipeline/renderers/__tests__/integration.test.js.map +1 -1
- package/dist/pipeline/renderers/html-clips.d.ts.map +1 -1
- package/dist/pipeline/renderers/html-clips.js +2 -0
- package/dist/pipeline/renderers/html-clips.js.map +1 -1
- package/dist/pipeline/renderers/html-template.d.ts.map +1 -1
- package/dist/pipeline/renderers/html-template.js +9 -1
- package/dist/pipeline/renderers/html-template.js.map +1 -1
- package/dist/pipeline/renderers/project-builder.d.ts.map +1 -1
- package/dist/pipeline/renderers/project-builder.js +28 -1
- package/dist/pipeline/renderers/project-builder.js.map +1 -1
- package/package.json +16 -13
- package/LICENSE +0 -21
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* Kling/Runway (video), FFmpeg (assembly/extension)
|
|
17
17
|
*/
|
|
18
18
|
import { readFile, writeFile, mkdir, unlink, rename } from "node:fs/promises";
|
|
19
|
-
import { resolve, basename, extname } from "node:path";
|
|
19
|
+
import { resolve, basename, dirname, extname } from "node:path";
|
|
20
20
|
import { existsSync } from "node:fs";
|
|
21
21
|
import { stringify as yamlStringify, parse as yamlParse } from "yaml";
|
|
22
22
|
import chalk from "chalk";
|
|
@@ -131,6 +131,27 @@ export async function extendVideoToTarget(videoPath, targetDuration, outputDir,
|
|
|
131
131
|
await unlink(videoPath);
|
|
132
132
|
await rename(extendedPath, videoPath);
|
|
133
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Log a provider failure to stderr. Centralizes the "what went wrong" output
|
|
136
|
+
* so call sites stop silently dropping provider errors into failedScenes.
|
|
137
|
+
* Safe to call with unknown errors, string messages, or VideoResult shapes.
|
|
138
|
+
*/
|
|
139
|
+
export function logSceneFailure(provider, sceneLabel, err) {
|
|
140
|
+
let msg;
|
|
141
|
+
if (err instanceof Error) {
|
|
142
|
+
msg = err.message;
|
|
143
|
+
}
|
|
144
|
+
else if (typeof err === "string") {
|
|
145
|
+
msg = err;
|
|
146
|
+
}
|
|
147
|
+
else if (err && typeof err === "object" && "error" in err && typeof err.error === "string") {
|
|
148
|
+
msg = err.error;
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
msg = String(err);
|
|
152
|
+
}
|
|
153
|
+
console.error(chalk.dim(`\n [${provider} ${sceneLabel}: ${msg}]`));
|
|
154
|
+
}
|
|
134
155
|
/**
|
|
135
156
|
* Generate video with retry logic for Grok provider
|
|
136
157
|
*/
|
|
@@ -149,10 +170,16 @@ export async function generateVideoWithRetryGrok(grok, segment, options, maxRetr
|
|
|
149
170
|
if (result.status !== "failed" && result.id) {
|
|
150
171
|
return { requestId: result.id };
|
|
151
172
|
}
|
|
173
|
+
// Provider returned status: "failed" with an error message — don't
|
|
174
|
+
// discard it. Surface via onProgress + stderr so the caller can see WHY.
|
|
175
|
+
const providerErr = result.error || "Grok returned failed status";
|
|
152
176
|
if (attempt < maxRetries) {
|
|
153
|
-
onProgress?.(`⚠
|
|
177
|
+
onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
|
|
154
178
|
await sleep(RETRY_DELAY_MS);
|
|
155
179
|
}
|
|
180
|
+
else {
|
|
181
|
+
console.error(chalk.dim(`\n [Grok error: ${providerErr}]`));
|
|
182
|
+
}
|
|
156
183
|
}
|
|
157
184
|
catch (err) {
|
|
158
185
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -192,10 +219,14 @@ export async function generateVideoWithRetryKling(kling, segment, options, maxRe
|
|
|
192
219
|
type: options.referenceImage ? "image2video" : "text2video",
|
|
193
220
|
};
|
|
194
221
|
}
|
|
222
|
+
const providerErr = result.error || "Kling returned failed status";
|
|
195
223
|
if (attempt < maxRetries) {
|
|
196
|
-
onProgress?.(`⚠
|
|
224
|
+
onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
|
|
197
225
|
await sleep(RETRY_DELAY_MS);
|
|
198
226
|
}
|
|
227
|
+
else {
|
|
228
|
+
console.error(chalk.dim(`\n [Kling error: ${providerErr}]`));
|
|
229
|
+
}
|
|
199
230
|
}
|
|
200
231
|
catch (err) {
|
|
201
232
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -204,7 +235,6 @@ export async function generateVideoWithRetryKling(kling, segment, options, maxRe
|
|
|
204
235
|
await sleep(RETRY_DELAY_MS);
|
|
205
236
|
}
|
|
206
237
|
else {
|
|
207
|
-
// Log the final error on last attempt
|
|
208
238
|
console.error(chalk.dim(`\n [Kling error: ${errMsg}]`));
|
|
209
239
|
}
|
|
210
240
|
}
|
|
@@ -226,10 +256,14 @@ export async function generateVideoWithRetryRunway(runway, segment, referenceIma
|
|
|
226
256
|
if (result.status !== "failed" && result.id) {
|
|
227
257
|
return { taskId: result.id };
|
|
228
258
|
}
|
|
259
|
+
const providerErr = result.error || "Runway returned failed status";
|
|
229
260
|
if (attempt < maxRetries) {
|
|
230
|
-
onProgress?.(`⚠
|
|
261
|
+
onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
|
|
231
262
|
await sleep(RETRY_DELAY_MS);
|
|
232
263
|
}
|
|
264
|
+
else {
|
|
265
|
+
console.error(chalk.dim(`\n [Runway error: ${providerErr}]`));
|
|
266
|
+
}
|
|
233
267
|
}
|
|
234
268
|
catch (err) {
|
|
235
269
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -263,10 +297,14 @@ export async function generateVideoWithRetryVeo(gemini, segment, options, maxRet
|
|
|
263
297
|
if (result.status !== "failed" && result.id) {
|
|
264
298
|
return { operationName: result.id };
|
|
265
299
|
}
|
|
300
|
+
const providerErr = result.error || "Veo returned failed status";
|
|
266
301
|
if (attempt < maxRetries) {
|
|
267
|
-
onProgress?.(`⚠
|
|
302
|
+
onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`);
|
|
268
303
|
await sleep(RETRY_DELAY_MS);
|
|
269
304
|
}
|
|
305
|
+
else {
|
|
306
|
+
console.error(chalk.dim(`\n [Veo error: ${providerErr}]`));
|
|
307
|
+
}
|
|
270
308
|
}
|
|
271
309
|
catch (err) {
|
|
272
310
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -461,14 +499,42 @@ export async function executeScriptToVideo(options) {
|
|
|
461
499
|
});
|
|
462
500
|
continue;
|
|
463
501
|
}
|
|
464
|
-
|
|
502
|
+
// Pre-TTS word-count heuristic (warn only; narration may still fit)
|
|
503
|
+
const wordCount = narrationText.split(/\s+/).filter(Boolean).length;
|
|
504
|
+
const maxWords = segment.duration > 5 ? 24 : 12;
|
|
505
|
+
if (wordCount > maxWords * 1.3) {
|
|
506
|
+
options.onProgress?.(`⚠ Scene ${i + 1} narration has ${wordCount} words (target ~${maxWords} for ${segment.duration}s); speech may rush.`);
|
|
507
|
+
}
|
|
508
|
+
options.onProgress?.(`Scene ${i + 1}/${segments.length}: generating narration...`);
|
|
509
|
+
let ttsResult = await elevenlabs.textToSpeech(narrationText, {
|
|
465
510
|
voiceId: options.voice,
|
|
466
511
|
});
|
|
467
512
|
if (ttsResult.success && ttsResult.audioBuffer) {
|
|
468
513
|
const audioPath = resolve(absOutputDir, `narration-${i + 1}.mp3`);
|
|
469
514
|
await writeFile(audioPath, ttsResult.audioBuffer);
|
|
470
515
|
// Get actual audio duration
|
|
471
|
-
|
|
516
|
+
let actualDuration = await getAudioDuration(audioPath);
|
|
517
|
+
// Auto speed-adjust if narration exceeds video bracket (5s or 10s).
|
|
518
|
+
// Pick the bracket from the pre-TTS segment.duration (the storyboard
|
|
519
|
+
// estimate), not actualDuration — actualDuration IS the overage.
|
|
520
|
+
const videoBracket = segment.duration > 5 ? 10 : 5;
|
|
521
|
+
const overageRatio = actualDuration / videoBracket;
|
|
522
|
+
if (overageRatio > 1.0 && overageRatio <= 1.35) {
|
|
523
|
+
const adjustedSpeed = Math.min(1.35, parseFloat(overageRatio.toFixed(2)));
|
|
524
|
+
options.onProgress?.(`Scene ${i + 1}: adjusting narration speed to ${adjustedSpeed}x...`);
|
|
525
|
+
const speedResult = await elevenlabs.textToSpeech(narrationText, {
|
|
526
|
+
voiceId: options.voice,
|
|
527
|
+
speed: adjustedSpeed,
|
|
528
|
+
});
|
|
529
|
+
if (speedResult.success && speedResult.audioBuffer) {
|
|
530
|
+
await writeFile(audioPath, speedResult.audioBuffer);
|
|
531
|
+
actualDuration = await getAudioDuration(audioPath);
|
|
532
|
+
ttsResult = speedResult;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
else if (overageRatio > 1.35) {
|
|
536
|
+
options.onProgress?.(`⚠ Scene ${i + 1} narration is ${((overageRatio - 1) * 100).toFixed(0)}% over target (${actualDuration.toFixed(1)}s vs ${videoBracket}s bracket)`);
|
|
537
|
+
}
|
|
472
538
|
segment.duration = actualDuration;
|
|
473
539
|
// Add to both arrays for backwards compatibility
|
|
474
540
|
result.narrations.push(audioPath);
|
|
@@ -528,6 +594,7 @@ export async function executeScriptToVideo(options) {
|
|
|
528
594
|
const imagePrompt = segment.visualStyle
|
|
529
595
|
? `${segment.visuals}. Style: ${segment.visualStyle}`
|
|
530
596
|
: segment.visuals;
|
|
597
|
+
options.onProgress?.(`Scene ${i + 1}/${segments.length}: generating image...`);
|
|
531
598
|
try {
|
|
532
599
|
let imageBuffer;
|
|
533
600
|
let imageUrl;
|
|
@@ -613,6 +680,7 @@ export async function executeScriptToVideo(options) {
|
|
|
613
680
|
}
|
|
614
681
|
const segment = segments[i];
|
|
615
682
|
const videoDuration = Math.min(15, Math.max(1, segment.duration));
|
|
683
|
+
options.onProgress?.(`Scene ${i + 1}/${segments.length}: generating video (grok)...`);
|
|
616
684
|
// Image-to-video: encode image as data URI
|
|
617
685
|
const imageBuffer = await readFile(imagePaths[i]);
|
|
618
686
|
const ext = extname(imagePaths[i]).toLowerCase().slice(1);
|
|
@@ -638,11 +706,13 @@ export async function executeScriptToVideo(options) {
|
|
|
638
706
|
result.videos.push(videoPath);
|
|
639
707
|
}
|
|
640
708
|
else {
|
|
709
|
+
logSceneFailure("Grok", `scene ${i + 1}`, waitResult);
|
|
641
710
|
videoPaths.push("");
|
|
642
711
|
result.failedScenes.push(i + 1);
|
|
643
712
|
}
|
|
644
713
|
}
|
|
645
|
-
catch {
|
|
714
|
+
catch (err) {
|
|
715
|
+
logSceneFailure("Grok", `scene ${i + 1}`, err);
|
|
646
716
|
videoPaths.push("");
|
|
647
717
|
result.failedScenes.push(i + 1);
|
|
648
718
|
}
|
|
@@ -659,6 +729,28 @@ export async function executeScriptToVideo(options) {
|
|
|
659
729
|
if (!kling.isConfigured()) {
|
|
660
730
|
return { success: false, outputDir: absOutputDir, scenes: segments.length, error: "Invalid Kling API key format. Use ACCESS_KEY:SECRET_KEY" };
|
|
661
731
|
}
|
|
732
|
+
// Kling image2video requires a public image URL (not base64). When an
|
|
733
|
+
// ImgBB key is configured, upload each scene image once and pass the
|
|
734
|
+
// URL as referenceImage; otherwise fall through to text2video.
|
|
735
|
+
const imgbbApiKey = (await getApiKeyFromConfig("imgbb")) || process.env.IMGBB_API_KEY;
|
|
736
|
+
const imageUrls = new Array(segments.length);
|
|
737
|
+
if (imgbbApiKey) {
|
|
738
|
+
options.onProgress?.("Uploading scene images for Kling image-to-video...");
|
|
739
|
+
for (let i = 0; i < imagePaths.length; i++) {
|
|
740
|
+
if (imagePaths[i]) {
|
|
741
|
+
try {
|
|
742
|
+
const imageBuffer = await readFile(imagePaths[i]);
|
|
743
|
+
const uploadResult = await uploadToImgbb(imageBuffer, imgbbApiKey);
|
|
744
|
+
if (uploadResult.success && uploadResult.url) {
|
|
745
|
+
imageUrls[i] = uploadResult.url;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
catch {
|
|
749
|
+
imageUrls[i] = undefined;
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
}
|
|
662
754
|
for (let i = 0; i < segments.length; i++) {
|
|
663
755
|
if (!imagePaths[i]) {
|
|
664
756
|
videoPaths.push("");
|
|
@@ -666,8 +758,12 @@ export async function executeScriptToVideo(options) {
|
|
|
666
758
|
}
|
|
667
759
|
const segment = segments[i];
|
|
668
760
|
const videoDuration = (segment.duration > 5 ? 10 : 5);
|
|
669
|
-
|
|
670
|
-
const taskResult = await generateVideoWithRetryKling(kling, segment, {
|
|
761
|
+
options.onProgress?.(`Scene ${i + 1}/${segments.length}: generating video (kling)...`);
|
|
762
|
+
const taskResult = await generateVideoWithRetryKling(kling, segment, {
|
|
763
|
+
duration: videoDuration,
|
|
764
|
+
aspectRatio: (options.aspectRatio || "16:9"),
|
|
765
|
+
referenceImage: imageUrls[i],
|
|
766
|
+
}, maxRetries);
|
|
671
767
|
if (taskResult) {
|
|
672
768
|
try {
|
|
673
769
|
const waitResult = await kling.waitForCompletion(taskResult.taskId, taskResult.type, undefined, 600000);
|
|
@@ -675,25 +771,25 @@ export async function executeScriptToVideo(options) {
|
|
|
675
771
|
const videoPath = resolve(absOutputDir, `scene-${i + 1}.mp4`);
|
|
676
772
|
const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
|
|
677
773
|
await writeFile(videoPath, buffer);
|
|
678
|
-
// Extend video to match narration duration
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
await rename(extendedPath, videoPath);
|
|
687
|
-
}
|
|
774
|
+
// Extend video to match narration duration. Use extendVideoToTarget
|
|
775
|
+
// so large gaps (ratio > 1.4) can use Kling's own extend API for
|
|
776
|
+
// natural continuation instead of freeze-frame padding.
|
|
777
|
+
await extendVideoToTarget(videoPath, segment.duration, absOutputDir, `Scene ${i + 1}`, {
|
|
778
|
+
kling,
|
|
779
|
+
videoId: waitResult.videoId,
|
|
780
|
+
onProgress: options.onProgress,
|
|
781
|
+
});
|
|
688
782
|
videoPaths.push(videoPath);
|
|
689
783
|
result.videos.push(videoPath);
|
|
690
784
|
}
|
|
691
785
|
else {
|
|
786
|
+
logSceneFailure("Kling", `scene ${i + 1}`, waitResult);
|
|
692
787
|
videoPaths.push("");
|
|
693
788
|
result.failedScenes.push(i + 1);
|
|
694
789
|
}
|
|
695
790
|
}
|
|
696
|
-
catch {
|
|
791
|
+
catch (err) {
|
|
792
|
+
logSceneFailure("Kling", `scene ${i + 1}`, err);
|
|
697
793
|
videoPaths.push("");
|
|
698
794
|
result.failedScenes.push(i + 1);
|
|
699
795
|
}
|
|
@@ -715,6 +811,7 @@ export async function executeScriptToVideo(options) {
|
|
|
715
811
|
}
|
|
716
812
|
const segment = segments[i];
|
|
717
813
|
const veoDuration = (segment.duration > 6 ? 8 : segment.duration > 4 ? 6 : 4);
|
|
814
|
+
options.onProgress?.(`Scene ${i + 1}/${segments.length}: generating video (veo)...`);
|
|
718
815
|
const taskResult = await generateVideoWithRetryVeo(veo, segment, { duration: veoDuration, aspectRatio: (options.aspectRatio || "16:9") }, maxRetries);
|
|
719
816
|
if (taskResult) {
|
|
720
817
|
try {
|
|
@@ -736,11 +833,13 @@ export async function executeScriptToVideo(options) {
|
|
|
736
833
|
result.videos.push(videoPath);
|
|
737
834
|
}
|
|
738
835
|
else {
|
|
836
|
+
logSceneFailure("Veo", `scene ${i + 1}`, waitResult);
|
|
739
837
|
videoPaths.push("");
|
|
740
838
|
result.failedScenes.push(i + 1);
|
|
741
839
|
}
|
|
742
840
|
}
|
|
743
|
-
catch {
|
|
841
|
+
catch (err) {
|
|
842
|
+
logSceneFailure("Veo", `scene ${i + 1}`, err);
|
|
744
843
|
videoPaths.push("");
|
|
745
844
|
result.failedScenes.push(i + 1);
|
|
746
845
|
}
|
|
@@ -767,6 +866,7 @@ export async function executeScriptToVideo(options) {
|
|
|
767
866
|
const referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`;
|
|
768
867
|
const videoDuration = (segment.duration > 5 ? 10 : 5);
|
|
769
868
|
const aspectRatio = options.aspectRatio === "1:1" ? "16:9" : (options.aspectRatio || "16:9");
|
|
869
|
+
options.onProgress?.(`Scene ${i + 1}/${segments.length}: generating video (runway)...`);
|
|
770
870
|
const taskResult = await generateVideoWithRetryRunway(runway, segment, referenceImage, { duration: videoDuration, aspectRatio }, maxRetries);
|
|
771
871
|
if (taskResult) {
|
|
772
872
|
try {
|
|
@@ -788,11 +888,13 @@ export async function executeScriptToVideo(options) {
|
|
|
788
888
|
result.videos.push(videoPath);
|
|
789
889
|
}
|
|
790
890
|
else {
|
|
891
|
+
logSceneFailure("Runway", `scene ${i + 1}`, waitResult);
|
|
791
892
|
videoPaths.push("");
|
|
792
893
|
result.failedScenes.push(i + 1);
|
|
793
894
|
}
|
|
794
895
|
}
|
|
795
|
-
catch {
|
|
896
|
+
catch (err) {
|
|
897
|
+
logSceneFailure("Runway", `scene ${i + 1}`, err);
|
|
796
898
|
videoPaths.push("");
|
|
797
899
|
result.failedScenes.push(i + 1);
|
|
798
900
|
}
|
|
@@ -912,8 +1014,15 @@ export async function executeScriptToVideo(options) {
|
|
|
912
1014
|
});
|
|
913
1015
|
currentTime += actualDuration;
|
|
914
1016
|
}
|
|
915
|
-
// Save project file
|
|
916
|
-
|
|
1017
|
+
// Save project file. Default is `{outputDir}/project.vibe.json`; callers
|
|
1018
|
+
// can override via projectFilePath (used by the CLI's `-o` handling).
|
|
1019
|
+
const projectPath = options.projectFilePath
|
|
1020
|
+
? resolve(process.cwd(), options.projectFilePath)
|
|
1021
|
+
: resolve(absOutputDir, "project.vibe.json");
|
|
1022
|
+
const projectParentDir = dirname(projectPath);
|
|
1023
|
+
if (!existsSync(projectParentDir)) {
|
|
1024
|
+
await mkdir(projectParentDir, { recursive: true });
|
|
1025
|
+
}
|
|
917
1026
|
await writeFile(projectPath, JSON.stringify(project.toJSON(), null, 2), "utf-8");
|
|
918
1027
|
result.projectPath = projectPath;
|
|
919
1028
|
result.totalDuration = currentTime;
|
|
@@ -995,6 +1104,8 @@ export async function executeRegenerateScene(options) {
|
|
|
995
1104
|
}
|
|
996
1105
|
}
|
|
997
1106
|
const regenerateVideo = options.videoOnly || (!options.narrationOnly && !options.imageOnly);
|
|
1107
|
+
const regenerateNarration = options.narrationOnly || (!options.videoOnly && !options.imageOnly);
|
|
1108
|
+
const regenerateImage = options.imageOnly || (!options.videoOnly && !options.narrationOnly);
|
|
998
1109
|
// Get API keys
|
|
999
1110
|
let videoApiKey;
|
|
1000
1111
|
if (regenerateVideo) {
|
|
@@ -1014,12 +1125,183 @@ export async function executeRegenerateScene(options) {
|
|
|
1014
1125
|
return { ...result, error: `${genInfo.name} API key required. Run 'vibe setup' or set ${genInfo.envVar} in .env` };
|
|
1015
1126
|
}
|
|
1016
1127
|
}
|
|
1128
|
+
let imageApiKey;
|
|
1129
|
+
if (regenerateImage) {
|
|
1130
|
+
const imageProvider = options.imageProvider || "openai";
|
|
1131
|
+
const imageKeyMap = {
|
|
1132
|
+
openai: { envVar: "OPENAI_API_KEY", name: "OpenAI" },
|
|
1133
|
+
gemini: { envVar: "GOOGLE_API_KEY", name: "Google" },
|
|
1134
|
+
grok: { envVar: "XAI_API_KEY", name: "xAI" },
|
|
1135
|
+
};
|
|
1136
|
+
const info = imageKeyMap[imageProvider];
|
|
1137
|
+
if (!info) {
|
|
1138
|
+
return { ...result, error: `Invalid imageProvider: ${imageProvider}` };
|
|
1139
|
+
}
|
|
1140
|
+
imageApiKey = (await getApiKey(info.envVar, info.name)) ?? undefined;
|
|
1141
|
+
if (!imageApiKey) {
|
|
1142
|
+
return { ...result, error: `${info.name} API key required. Run 'vibe setup' or set ${info.envVar} in .env` };
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
let elevenlabsApiKey;
|
|
1146
|
+
if (regenerateNarration) {
|
|
1147
|
+
elevenlabsApiKey = (await getApiKey("ELEVENLABS_API_KEY", "ElevenLabs")) ?? undefined;
|
|
1148
|
+
if (!elevenlabsApiKey) {
|
|
1149
|
+
return { ...result, error: "ElevenLabs API key required. Run 'vibe setup' or set ELEVENLABS_API_KEY in .env" };
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
// Track storyboard mutations so we only rewrite when segment durations change.
|
|
1153
|
+
let storyboardMutated = false;
|
|
1017
1154
|
// Process each scene
|
|
1018
1155
|
for (const sceneNum of options.scenes) {
|
|
1019
1156
|
const segment = segments[sceneNum - 1];
|
|
1157
|
+
const narrationPath = resolve(outputDir, `narration-${sceneNum}.mp3`);
|
|
1020
1158
|
const imagePath = resolve(outputDir, `scene-${sceneNum}.png`);
|
|
1021
1159
|
const videoPath = resolve(outputDir, `scene-${sceneNum}.mp4`);
|
|
1160
|
+
let sceneFailed = false;
|
|
1161
|
+
// Narration regeneration
|
|
1162
|
+
if (regenerateNarration && elevenlabsApiKey) {
|
|
1163
|
+
options.onProgress?.(`Scene ${sceneNum}: regenerating narration...`);
|
|
1164
|
+
const elevenlabs = new ElevenLabsProvider();
|
|
1165
|
+
await elevenlabs.initialize({ apiKey: elevenlabsApiKey });
|
|
1166
|
+
const narrationText = segment.narration || segment.description;
|
|
1167
|
+
const ttsResult = await elevenlabs.textToSpeech(narrationText, {
|
|
1168
|
+
voiceId: options.voice,
|
|
1169
|
+
});
|
|
1170
|
+
if (ttsResult.success && ttsResult.audioBuffer) {
|
|
1171
|
+
await writeFile(narrationPath, ttsResult.audioBuffer);
|
|
1172
|
+
segment.duration = await getAudioDuration(narrationPath);
|
|
1173
|
+
storyboardMutated = true;
|
|
1174
|
+
}
|
|
1175
|
+
else {
|
|
1176
|
+
sceneFailed = true;
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
// Image regeneration (with character-consistency reference when available)
|
|
1180
|
+
if (!sceneFailed && regenerateImage && imageApiKey) {
|
|
1181
|
+
options.onProgress?.(`Scene ${sceneNum}: regenerating image...`);
|
|
1182
|
+
const imageProvider = options.imageProvider || "openai";
|
|
1183
|
+
const characterDesc = segment.characterDescription || segments[0]?.characterDescription;
|
|
1184
|
+
let imagePrompt = segment.visualStyle
|
|
1185
|
+
? `${segment.visuals}. Style: ${segment.visualStyle}`
|
|
1186
|
+
: segment.visuals;
|
|
1187
|
+
if (characterDesc) {
|
|
1188
|
+
imagePrompt = `${imagePrompt}\n\nIMPORTANT - Character appearance must match exactly: ${characterDesc}`;
|
|
1189
|
+
}
|
|
1190
|
+
// Pick a reference image for character consistency: caller-specified,
|
|
1191
|
+
// else auto-detect the first existing scene image that isn't this scene.
|
|
1192
|
+
let referenceImageBuffer;
|
|
1193
|
+
const refSceneNum = options.referenceScene;
|
|
1194
|
+
if (refSceneNum && refSceneNum >= 1 && refSceneNum <= segments.length && refSceneNum !== sceneNum) {
|
|
1195
|
+
const refImagePath = resolve(outputDir, `scene-${refSceneNum}.png`);
|
|
1196
|
+
if (existsSync(refImagePath)) {
|
|
1197
|
+
referenceImageBuffer = await readFile(refImagePath);
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
else if (!refSceneNum) {
|
|
1201
|
+
for (let i = 1; i <= segments.length; i++) {
|
|
1202
|
+
if (i !== sceneNum) {
|
|
1203
|
+
const otherImagePath = resolve(outputDir, `scene-${i}.png`);
|
|
1204
|
+
if (existsSync(otherImagePath)) {
|
|
1205
|
+
referenceImageBuffer = await readFile(otherImagePath);
|
|
1206
|
+
break;
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
const dalleImageSizes = {
|
|
1212
|
+
"16:9": "1536x1024",
|
|
1213
|
+
"9:16": "1024x1536",
|
|
1214
|
+
"1:1": "1024x1024",
|
|
1215
|
+
};
|
|
1216
|
+
let imageBuffer;
|
|
1217
|
+
let imageUrl;
|
|
1218
|
+
if (imageProvider === "openai") {
|
|
1219
|
+
const openaiImage = new OpenAIImageProvider();
|
|
1220
|
+
await openaiImage.initialize({ apiKey: imageApiKey });
|
|
1221
|
+
const imageResult = await openaiImage.generateImage(imagePrompt, {
|
|
1222
|
+
size: dalleImageSizes[options.aspectRatio || "16:9"] || "1536x1024",
|
|
1223
|
+
quality: "standard",
|
|
1224
|
+
});
|
|
1225
|
+
if (imageResult.success && imageResult.images?.[0]) {
|
|
1226
|
+
const img = imageResult.images[0];
|
|
1227
|
+
if (img.base64)
|
|
1228
|
+
imageBuffer = Buffer.from(img.base64, "base64");
|
|
1229
|
+
else if (img.url)
|
|
1230
|
+
imageUrl = img.url;
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
else if (imageProvider === "gemini") {
|
|
1234
|
+
const gemini = new GeminiProvider();
|
|
1235
|
+
await gemini.initialize({ apiKey: imageApiKey });
|
|
1236
|
+
if (referenceImageBuffer) {
|
|
1237
|
+
// Gemini edit-with-reference path — pulls the character forward
|
|
1238
|
+
// while describing the new action. Simplified visuals pick the
|
|
1239
|
+
// first recognised action verb to reduce prompt bleed.
|
|
1240
|
+
const simplifiedVisuals = segment.visuals.split(/[,.]/).find((part) => part.includes("standing") || part.includes("sitting") || part.includes("walking") ||
|
|
1241
|
+
part.includes("lying") || part.includes("reaching") || part.includes("looking") ||
|
|
1242
|
+
part.includes("working") || part.includes("coding") || part.includes("typing")) || segment.visuals.split(".")[0];
|
|
1243
|
+
const editPrompt = `Generate a new image showing the SAME SINGLE person from the reference image in a new scene.
|
|
1244
|
+
|
|
1245
|
+
REFERENCE: Look at the person in the reference image - their face, hair, build, and overall appearance.
|
|
1246
|
+
|
|
1247
|
+
NEW SCENE: ${simplifiedVisuals}
|
|
1248
|
+
|
|
1249
|
+
CRITICAL RULES:
|
|
1250
|
+
1. Show ONLY ONE person - the exact same individual from the reference image
|
|
1251
|
+
2. The person must have the IDENTICAL face, hair style, and body type
|
|
1252
|
+
3. Do NOT show multiple people or duplicate the character
|
|
1253
|
+
4. Create a single moment in time, one pose, one action
|
|
1254
|
+
5. Match the art style and quality of the reference image
|
|
1255
|
+
|
|
1256
|
+
Generate the single-person scene image now.`;
|
|
1257
|
+
const imageResult = await gemini.editImage([referenceImageBuffer], editPrompt, {
|
|
1258
|
+
aspectRatio: (options.aspectRatio || "16:9"),
|
|
1259
|
+
});
|
|
1260
|
+
if (imageResult.success && imageResult.images?.[0]?.base64) {
|
|
1261
|
+
imageBuffer = Buffer.from(imageResult.images[0].base64, "base64");
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
else {
|
|
1265
|
+
const imageResult = await gemini.generateImage(imagePrompt, {
|
|
1266
|
+
aspectRatio: (options.aspectRatio || "16:9"),
|
|
1267
|
+
});
|
|
1268
|
+
if (imageResult.success && imageResult.images?.[0]?.base64) {
|
|
1269
|
+
imageBuffer = Buffer.from(imageResult.images[0].base64, "base64");
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
else if (imageProvider === "grok") {
|
|
1274
|
+
const grok = new GrokProvider();
|
|
1275
|
+
await grok.initialize({ apiKey: imageApiKey });
|
|
1276
|
+
const imageResult = await grok.generateImage(imagePrompt, {
|
|
1277
|
+
aspectRatio: options.aspectRatio || "16:9",
|
|
1278
|
+
});
|
|
1279
|
+
if (imageResult.success && imageResult.images?.[0]) {
|
|
1280
|
+
const img = imageResult.images[0];
|
|
1281
|
+
if (img.base64)
|
|
1282
|
+
imageBuffer = Buffer.from(img.base64, "base64");
|
|
1283
|
+
else if (img.url)
|
|
1284
|
+
imageUrl = img.url;
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
if (imageBuffer) {
|
|
1288
|
+
await writeFile(imagePath, imageBuffer);
|
|
1289
|
+
}
|
|
1290
|
+
else if (imageUrl) {
|
|
1291
|
+
const response = await fetch(imageUrl);
|
|
1292
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
1293
|
+
await writeFile(imagePath, buffer);
|
|
1294
|
+
}
|
|
1295
|
+
else {
|
|
1296
|
+
sceneFailed = true;
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
if (sceneFailed) {
|
|
1300
|
+
result.failedScenes.push(sceneNum);
|
|
1301
|
+
continue;
|
|
1302
|
+
}
|
|
1022
1303
|
if (regenerateVideo && videoApiKey) {
|
|
1304
|
+
options.onProgress?.(`Scene ${sceneNum}: regenerating video (${options.generator || "grok"})...`);
|
|
1023
1305
|
if (!existsSync(imagePath)) {
|
|
1024
1306
|
result.failedScenes.push(sceneNum);
|
|
1025
1307
|
continue;
|
|
@@ -1057,10 +1339,12 @@ export async function executeRegenerateScene(options) {
|
|
|
1057
1339
|
result.regeneratedScenes.push(sceneNum);
|
|
1058
1340
|
}
|
|
1059
1341
|
else {
|
|
1342
|
+
logSceneFailure("Grok", `scene ${sceneNum}`, waitResult);
|
|
1060
1343
|
result.failedScenes.push(sceneNum);
|
|
1061
1344
|
}
|
|
1062
1345
|
}
|
|
1063
|
-
catch {
|
|
1346
|
+
catch (err) {
|
|
1347
|
+
logSceneFailure("Grok", `scene ${sceneNum}`, err);
|
|
1064
1348
|
result.failedScenes.push(sceneNum);
|
|
1065
1349
|
}
|
|
1066
1350
|
}
|
|
@@ -1098,10 +1382,12 @@ export async function executeRegenerateScene(options) {
|
|
|
1098
1382
|
result.regeneratedScenes.push(sceneNum);
|
|
1099
1383
|
}
|
|
1100
1384
|
else {
|
|
1385
|
+
logSceneFailure("Veo", `scene ${sceneNum}`, waitResult);
|
|
1101
1386
|
result.failedScenes.push(sceneNum);
|
|
1102
1387
|
}
|
|
1103
1388
|
}
|
|
1104
|
-
catch {
|
|
1389
|
+
catch (err) {
|
|
1390
|
+
logSceneFailure("Veo", `scene ${sceneNum}`, err);
|
|
1105
1391
|
result.failedScenes.push(sceneNum);
|
|
1106
1392
|
}
|
|
1107
1393
|
}
|
|
@@ -1136,22 +1422,22 @@ export async function executeRegenerateScene(options) {
|
|
|
1136
1422
|
if (waitResult.status === "completed" && waitResult.videoUrl) {
|
|
1137
1423
|
const buffer = await downloadVideo(waitResult.videoUrl, videoApiKey);
|
|
1138
1424
|
await writeFile(videoPath, buffer);
|
|
1139
|
-
// Extend
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
await rename(extendedPath, videoPath);
|
|
1147
|
-
}
|
|
1425
|
+
// Extend via extendVideoToTarget so Kling's own extend API can
|
|
1426
|
+
// be used for large gaps (ratio > 1.4) instead of freeze frames.
|
|
1427
|
+
await extendVideoToTarget(videoPath, segment.duration, outputDir, `Scene ${sceneNum}`, {
|
|
1428
|
+
kling,
|
|
1429
|
+
videoId: waitResult.videoId,
|
|
1430
|
+
onProgress: options.onProgress,
|
|
1431
|
+
});
|
|
1148
1432
|
result.regeneratedScenes.push(sceneNum);
|
|
1149
1433
|
}
|
|
1150
1434
|
else {
|
|
1435
|
+
logSceneFailure("Kling", `scene ${sceneNum}`, waitResult);
|
|
1151
1436
|
result.failedScenes.push(sceneNum);
|
|
1152
1437
|
}
|
|
1153
1438
|
}
|
|
1154
|
-
catch {
|
|
1439
|
+
catch (err) {
|
|
1440
|
+
logSceneFailure("Kling", `scene ${sceneNum}`, err);
|
|
1155
1441
|
result.failedScenes.push(sceneNum);
|
|
1156
1442
|
}
|
|
1157
1443
|
}
|
|
@@ -1186,10 +1472,12 @@ export async function executeRegenerateScene(options) {
|
|
|
1186
1472
|
result.regeneratedScenes.push(sceneNum);
|
|
1187
1473
|
}
|
|
1188
1474
|
else {
|
|
1475
|
+
logSceneFailure("Runway", `scene ${sceneNum}`, waitResult);
|
|
1189
1476
|
result.failedScenes.push(sceneNum);
|
|
1190
1477
|
}
|
|
1191
1478
|
}
|
|
1192
|
-
catch {
|
|
1479
|
+
catch (err) {
|
|
1480
|
+
logSceneFailure("Runway", `scene ${sceneNum}`, err);
|
|
1193
1481
|
result.failedScenes.push(sceneNum);
|
|
1194
1482
|
}
|
|
1195
1483
|
}
|
|
@@ -1198,6 +1486,23 @@ export async function executeRegenerateScene(options) {
|
|
|
1198
1486
|
}
|
|
1199
1487
|
}
|
|
1200
1488
|
}
|
|
1489
|
+
else if (!sceneFailed) {
|
|
1490
|
+
// narration-only / image-only / both: no video pass, so record success here
|
|
1491
|
+
result.regeneratedScenes.push(sceneNum);
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
// Persist storyboard if narration regeneration changed segment durations.
|
|
1495
|
+
// Preserve the on-disk format (yaml vs json) so no silent downgrades.
|
|
1496
|
+
if (storyboardMutated) {
|
|
1497
|
+
let currentTime = 0;
|
|
1498
|
+
for (const segment of segments) {
|
|
1499
|
+
segment.startTime = currentTime;
|
|
1500
|
+
currentTime += segment.duration;
|
|
1501
|
+
}
|
|
1502
|
+
const serialized = storyboardPath.endsWith(".yaml")
|
|
1503
|
+
? yamlStringify({ scenes: segments }, { indent: 2 })
|
|
1504
|
+
: JSON.stringify(segments, null, 2);
|
|
1505
|
+
await writeFile(storyboardPath, serialized, "utf-8");
|
|
1201
1506
|
}
|
|
1202
1507
|
result.success = result.failedScenes.length === 0;
|
|
1203
1508
|
return result;
|