@effectnode/media 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/backend/movie-backend/agent/prompt/ltx.txt +20 -0
  2. package/dist/backend/movie-backend/agent/prompt/script.md +111 -1
  3. package/dist/backend/movie-backend/core.js +41 -0
  4. package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
  5. package/dist/backend/movie-backend/generation-queue.js +75 -8
  6. package/dist/backend/movie-backend/render-media.d.ts +79 -2
  7. package/dist/backend/movie-backend/render-media.js +694 -418
  8. package/frontend/src/movie-app/components/EditorTabs/AdvancedVoiceCloneTab.tsx +366 -0
  9. package/frontend/src/movie-app/components/EditorTabs/AudioToVideoTab.tsx +406 -0
  10. package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +47 -0
  11. package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +155 -1
  12. package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +33 -13
  13. package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +26 -5
  14. package/frontend/src/movie-app/components/EditorTabs/UpscaleTab.tsx +343 -0
  15. package/frontend/src/movie-app/components/EditorTabs/VoiceCloneTab.tsx +365 -0
  16. package/frontend/src/movie-app/components/ProjectEditorPage.tsx +130 -125
  17. package/frontend/src/movie-app/stores/advancedVoiceCloneStore.ts +205 -0
  18. package/frontend/src/movie-app/stores/aiModelStore.ts +25 -2
  19. package/frontend/src/movie-app/stores/audioToVideoStore.ts +274 -0
  20. package/frontend/src/movie-app/stores/generationStore.ts +156 -255
  21. package/frontend/src/movie-app/stores/movieStudioStore.ts +10 -3
  22. package/frontend/src/movie-app/stores/queueStore.ts +47 -1
  23. package/frontend/src/movie-app/stores/upscaleStore.ts +118 -0
  24. package/frontend/src/movie-app/stores/voiceCloneStore.ts +227 -0
  25. package/package.json +1 -1
  26. package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +0 -913
  27. package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +0 -305
  28. package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +0 -249
  29. package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +0 -267
  30. package/frontend/src/movie-app/lib/batchVoiceStorage.ts +0 -75
  31. package/frontend/src/movie-app/stores/batchVoiceStore.ts +0 -990
  32. package/frontend/src/movie-app/stores/sceneVisualStore.ts +0 -251
@@ -1,990 +0,0 @@
1
- import { create } from "zustand";
2
- import type {
3
- AspectRatio,
4
- Resolution,
5
- VideoMode,
6
- ProjectImage,
7
- } from "./generationStore";
8
- import { loadFFmpeg } from "../lib/ffmpeg";
9
- import {
10
- clearBatchVoiceState,
11
- loadBatchVoiceState,
12
- saveBatchVoiceState,
13
- type PersistedBatchVoiceState,
14
- type VoiceQuality,
15
- } from "../lib/batchVoiceStorage";
16
-
17
- const API_BASE = `http://localhost:${(window as any).PORT}`;
18
-
19
- export type BatchVoiceRowStatus =
20
- | "idle"
21
- | "uploading"
22
- | "tts"
23
- | "video"
24
- | "muxing"
25
- | "done"
26
- | "error";
27
-
28
- export interface BatchVoiceRow {
29
- id: string;
30
- // Video prompt (drives the image-to-video stage)
31
- prompt: string;
32
- // TTS text (spoken in the cloned voice). Empty = silent video (no voiceover).
33
- script: string;
34
- imagePath: string | null;
35
- imageUrl: string | null;
36
- imageFilename: string | null;
37
- status: BatchVoiceRowStatus;
38
- result: string | null;
39
- error: string | null;
40
- logs: string[];
41
- audioResult: string | null;
42
- motionResult: string | null;
43
- }
44
-
45
- interface BatchVoiceStore {
46
- // Rows
47
- rows: BatchVoiceRow[];
48
-
49
- // Shared video settings
50
- duration: number;
51
- aspectRatio: AspectRatio;
52
- resolution: Resolution;
53
- mode: VideoMode;
54
-
55
- // Shared voice settings
56
- quality: VoiceQuality;
57
- voiceRefPath: string | null;
58
- voiceRefUrl: string | null;
59
- voiceRefFilename: string | null;
60
-
61
- // Batch generation state
62
- running: boolean;
63
- progress: { current: number; total: number } | null;
64
- cancelRequested: boolean;
65
- logs: string[];
66
-
67
- // Stitching state
68
- stitching: boolean;
69
- stitchLogs: string[];
70
- stitchResult: string | null;
71
- stitchError: string | null;
72
-
73
- // Persistence
74
- projectId: string | null;
75
- hydrated: boolean;
76
- hydrate: (projectId: string) => Promise<void>;
77
- clear: () => void;
78
-
79
- addRow: () => void;
80
- removeRow: (id: string) => void;
81
- updatePrompt: (id: string, prompt: string) => void;
82
- updateScript: (id: string, script: string) => void;
83
- clearRowResult: (id: string) => void;
84
- uploadRowImage: (
85
- id: string,
86
- base64: string,
87
- filename: string | undefined,
88
- projectId: string,
89
- ) => Promise<string | null>;
90
- setRowImage: (id: string, image: ProjectImage) => void;
91
-
92
- setDuration: (v: number) => void;
93
- setAspectRatio: (v: AspectRatio) => void;
94
- setResolution: (v: Resolution) => void;
95
- setMode: (v: VideoMode) => void;
96
- setQuality: (v: VoiceQuality) => void;
97
- uploadVoiceRef: (
98
- base64: string,
99
- filename: string,
100
- projectId: string,
101
- ) => Promise<string | null>;
102
-
103
- generateRows: (projectId: string, ids: string[]) => Promise<void>;
104
- generateRow: (projectId: string, id: string) => Promise<void>;
105
- generateAll: (projectId: string) => Promise<void>;
106
- cancel: () => void;
107
-
108
- stitchVideos: () => Promise<void>;
109
-
110
- reset: () => void;
111
- }
112
-
113
- let batchAbortController: AbortController | null = null;
114
-
115
- function getDimensions(
116
- aspect: AspectRatio,
117
- resolution: Resolution,
118
- ): { width: number; height: number } {
119
- const size = parseInt(resolution);
120
- switch (aspect) {
121
- case "1:1":
122
- return { width: size, height: size };
123
- case "16:9":
124
- return { width: Math.round((size * 16) / 9), height: size };
125
- case "9:16":
126
- return { width: size, height: Math.round((size * 16) / 9) };
127
- case "4:3":
128
- return { width: Math.round((size * 4) / 3), height: size };
129
- case "3:4":
130
- return { width: size, height: Math.round((size * 4) / 3) };
131
- }
132
- }
133
-
134
- // The image-to-video backend only accepts a bare filename, so normalise any
135
- // uploaded path / file URL / api-files URL down to just the basename.
136
- function normalizeImagePath(path: string): string {
137
- let p = path;
138
- if (p.includes("/api/files?path=")) {
139
- try {
140
- const url = new URL(p);
141
- p = url.searchParams.get("path") || p;
142
- } catch {
143
- // not a valid URL, use as-is
144
- }
145
- }
146
- if (p.startsWith("file://")) {
147
- p = p.slice(7);
148
- }
149
- return p.split("/").pop() || p;
150
- }
151
-
152
- async function readSSEStream(
153
- response: Response,
154
- onEvent: (event: string, data: any) => void,
155
- ): Promise<void> {
156
- const reader = response.body?.getReader();
157
- if (!reader) return;
158
-
159
- const decoder = new TextDecoder();
160
- let buffer = "";
161
-
162
- try {
163
- while (true) {
164
- const { done, value } = await reader.read();
165
- if (done) break;
166
-
167
- buffer += decoder.decode(value, { stream: true });
168
- const lines = buffer.split("\n");
169
- buffer = lines.pop() || "";
170
-
171
- let eventType = "message";
172
- for (const line of lines) {
173
- if (line.startsWith("event: ")) {
174
- eventType = line.slice(7).trim();
175
- } else if (line.startsWith("data: ")) {
176
- try {
177
- onEvent(eventType, JSON.parse(line.slice(6)));
178
- } catch {
179
- // skip malformed lines
180
- }
181
- eventType = "message";
182
- }
183
- }
184
- }
185
- } finally {
186
- reader.releaseLock();
187
- }
188
- }
189
-
190
- function playBeep() {
191
- try {
192
- const ctx = new (
193
- window.AudioContext || (window as any).webkitAudioContext
194
- )();
195
- const osc = ctx.createOscillator();
196
- const gain = ctx.createGain();
197
- osc.connect(gain);
198
- gain.connect(ctx.destination);
199
- osc.type = "sine";
200
- osc.frequency.setValueAtTime(880, ctx.currentTime);
201
- gain.gain.setValueAtTime(0.3, ctx.currentTime);
202
- gain.gain.exponentialRampToValueAtTime(0.01, ctx.currentTime + 0.4);
203
- osc.start(ctx.currentTime);
204
- osc.stop(ctx.currentTime + 0.4);
205
- } catch {
206
- // silently ignore if audio not available
207
- }
208
- }
209
-
210
- function makeRow(): BatchVoiceRow {
211
- return {
212
- id: `row-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`,
213
- prompt: "",
214
- script: "",
215
- imagePath: null,
216
- imageUrl: null,
217
- imageFilename: null,
218
- status: "idle",
219
- result: null,
220
- error: null,
221
- logs: [],
222
- audioResult: null,
223
- motionResult: null,
224
- };
225
- }
226
-
227
- function toPersistedState(
228
- s: Pick<
229
- BatchVoiceStore,
230
- | "rows"
231
- | "duration"
232
- | "aspectRatio"
233
- | "resolution"
234
- | "mode"
235
- | "quality"
236
- | "voiceRefPath"
237
- | "voiceRefFilename"
238
- >,
239
- ): PersistedBatchVoiceState {
240
- return {
241
- rows: s.rows.map((r) => ({
242
- id: r.id,
243
- prompt: r.prompt,
244
- script: r.script,
245
- imagePath: r.imagePath,
246
- imageUrl: r.imageUrl,
247
- imageFilename: r.imageFilename,
248
- })),
249
- duration: s.duration,
250
- aspectRatio: s.aspectRatio,
251
- resolution: s.resolution,
252
- mode: s.mode,
253
- quality: s.quality,
254
- voiceRefPath: s.voiceRefPath,
255
- voiceRefFilename: s.voiceRefFilename,
256
- };
257
- }
258
-
259
- // Fire-and-forget save of the current editable UI state.
260
- function persistBatchState() {
261
- const { projectId } = useBatchVoiceStore.getState();
262
- if (!projectId) return;
263
- void saveBatchVoiceState(
264
- projectId,
265
- toPersistedState(useBatchVoiceStore.getState()),
266
- );
267
- }
268
-
269
- export const useBatchVoiceStore = create<BatchVoiceStore>((set, get) => ({
270
- rows: [makeRow()],
271
-
272
- duration: 5,
273
- aspectRatio: "1:1",
274
- resolution: "480p",
275
- mode: "distilled",
276
-
277
- quality: "high",
278
- voiceRefPath: null,
279
- voiceRefUrl: null,
280
- voiceRefFilename: null,
281
-
282
- running: false,
283
- progress: null,
284
- cancelRequested: false,
285
- logs: [],
286
-
287
- stitching: false,
288
- stitchLogs: [],
289
- stitchResult: null,
290
- stitchError: null,
291
-
292
- projectId: null,
293
- hydrated: false,
294
-
295
- hydrate: async (projectId) => {
296
- // No-op if we've already hydrated for this project.
297
- if (get().hydrated && get().projectId === projectId) return;
298
-
299
- // Switching to a different project: reset to defaults so rows and
300
- // settings from the previous project don't leak through.
301
- const previous = get().projectId;
302
- if (previous !== null && previous !== projectId) {
303
- get().reset();
304
- }
305
- set({ hydrated: true, projectId });
306
-
307
- const stored = await loadBatchVoiceState(projectId);
308
- if (!stored) return;
309
-
310
- set((s) => {
311
- const rows: BatchVoiceRow[] = (stored.rows ?? []).map((r) => ({
312
- id: r.id,
313
- prompt: r.prompt,
314
- script: r.script,
315
- imagePath: r.imagePath,
316
- imageUrl: r.imageUrl,
317
- imageFilename: r.imageFilename,
318
- status: "idle",
319
- result: null,
320
- error: null,
321
- logs: [],
322
- audioResult: null,
323
- motionResult: null,
324
- }));
325
-
326
- // Rebuild the preview URL for the stored voice reference so the audio
327
- // player keeps working after a reload.
328
- const restoredRefPath = stored.voiceRefPath ?? s.voiceRefPath;
329
-
330
- return {
331
- rows: rows.length > 0 ? rows : [makeRow()],
332
- duration: stored.duration ?? s.duration,
333
- aspectRatio: stored.aspectRatio ?? s.aspectRatio,
334
- resolution: stored.resolution ?? s.resolution,
335
- mode: stored.mode ?? s.mode,
336
- quality: stored.quality ?? s.quality,
337
- voiceRefPath: restoredRefPath,
338
- voiceRefUrl: restoredRefPath
339
- ? `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(restoredRefPath)}`
340
- : null,
341
- voiceRefFilename: stored.voiceRefFilename ?? s.voiceRefFilename,
342
- };
343
- });
344
- },
345
-
346
- clear: () => {
347
- const { projectId } = get();
348
- get().reset();
349
- if (projectId) void clearBatchVoiceState(projectId);
350
- },
351
-
352
- addRow: () => {
353
- set((s) => ({ rows: [...s.rows, makeRow()] }));
354
- persistBatchState();
355
- },
356
-
357
- removeRow: (id) => {
358
- set((s) => ({ rows: s.rows.filter((r) => r.id !== id) }));
359
- persistBatchState();
360
- },
361
-
362
- updatePrompt: (id, prompt) => {
363
- set((s) => ({
364
- rows: s.rows.map((r) => (r.id === id ? { ...r, prompt } : r)),
365
- }));
366
- persistBatchState();
367
- },
368
-
369
- updateScript: (id, script) => {
370
- set((s) => ({
371
- rows: s.rows.map((r) => (r.id === id ? { ...r, script } : r)),
372
- }));
373
- persistBatchState();
374
- },
375
-
376
- clearRowResult: (id) =>
377
- set((s) => ({
378
- rows: s.rows.map((r) =>
379
- r.id === id
380
- ? {
381
- ...r,
382
- result: null,
383
- error: null,
384
- status: "idle",
385
- audioResult: null,
386
- motionResult: null,
387
- }
388
- : r,
389
- ),
390
- })),
391
-
392
- uploadRowImage: async (id, base64, filename, projectId) => {
393
- set((s) => ({
394
- rows: s.rows.map((r) =>
395
- r.id === id ? { ...r, status: "uploading" } : r,
396
- ),
397
- }));
398
-
399
- try {
400
- const res = await fetch(`${API_BASE}/api/upload/image`, {
401
- method: "POST",
402
- headers: { "Content-Type": "application/json" },
403
- body: JSON.stringify({
404
- image: base64,
405
- filename: filename || `upload-${Date.now()}.png`,
406
- projectId,
407
- }),
408
- });
409
-
410
- if (!res.ok) {
411
- const err = await res.text();
412
- set((s) => ({
413
- rows: s.rows.map((r) =>
414
- r.id === id ? { ...r, status: "error", error: err } : r,
415
- ),
416
- }));
417
- return null;
418
- }
419
-
420
- const data = await res.json();
421
- const url = `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`;
422
- set((s) => ({
423
- rows: s.rows.map((r) =>
424
- r.id === id
425
- ? {
426
- ...r,
427
- status: "idle",
428
- imagePath: data.path,
429
- imageUrl: url,
430
- imageFilename: data.filename,
431
- }
432
- : r,
433
- ),
434
- }));
435
- persistBatchState();
436
- return data.path as string;
437
- } catch (e) {
438
- set((s) => ({
439
- rows: s.rows.map((r) =>
440
- r.id === id ? { ...r, status: "error", error: String(e) } : r,
441
- ),
442
- }));
443
- return null;
444
- }
445
- },
446
-
447
- setRowImage: (id, image) => {
448
- set((s) => ({
449
- rows: s.rows.map((r) =>
450
- r.id === id
451
- ? {
452
- ...r,
453
- imagePath: image.url,
454
- imageUrl: image.url,
455
- imageFilename: image.filename,
456
- status: "idle",
457
- error: null,
458
- }
459
- : r,
460
- ),
461
- }));
462
- persistBatchState();
463
- },
464
-
465
- setDuration: (duration) => {
466
- set({ duration });
467
- persistBatchState();
468
- },
469
- setAspectRatio: (aspectRatio) => {
470
- set({ aspectRatio });
471
- persistBatchState();
472
- },
473
- setResolution: (resolution) => {
474
- set({ resolution });
475
- persistBatchState();
476
- },
477
- setMode: (mode) => {
478
- set({ mode });
479
- persistBatchState();
480
- },
481
- setQuality: (quality) => {
482
- set({ quality });
483
- persistBatchState();
484
- },
485
-
486
- uploadVoiceRef: async (base64, filename, projectId) => {
487
- try {
488
- const res = await fetch(`${API_BASE}/api/upload/audio`, {
489
- method: "POST",
490
- headers: { "Content-Type": "application/json" },
491
- body: JSON.stringify({
492
- audio: base64,
493
- filename,
494
- projectId,
495
- }),
496
- });
497
-
498
- if (!res.ok) {
499
- return null;
500
- }
501
-
502
- const data = await res.json();
503
- set({
504
- voiceRefPath: data.path,
505
- voiceRefUrl: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
506
- voiceRefFilename: data.filename,
507
- });
508
- persistBatchState();
509
- return data.path as string;
510
- } catch {
511
- return null;
512
- }
513
- },
514
-
515
- generateRows: async (projectId, ids) => {
516
- if (get().running) return;
517
-
518
- const rows = get().rows;
519
- const targets = ids
520
- .map((id) => rows.find((r) => r.id === id))
521
- .filter((r): r is BatchVoiceRow => !!r)
522
- .filter(
523
- (r) =>
524
- r.prompt.trim() &&
525
- r.imagePath &&
526
- (!r.script.trim() || get().voiceRefPath),
527
- );
528
-
529
- if (targets.length === 0) return;
530
-
531
- const { duration, aspectRatio, resolution, mode, quality, voiceRefPath } =
532
- get();
533
- const { width, height } = getDimensions(aspectRatio, resolution);
534
- const refAudioPath = voiceRefPath ? normalizeImagePath(voiceRefPath) : null;
535
-
536
- batchAbortController = new AbortController();
537
- const signal = batchAbortController.signal;
538
-
539
- set({
540
- running: true,
541
- progress: { current: 0, total: targets.length },
542
- cancelRequested: false,
543
- logs: [],
544
- });
545
-
546
- let cancelled = false;
547
-
548
- for (let i = 0; i < targets.length; i++) {
549
- if (get().cancelRequested) {
550
- cancelled = true;
551
- break;
552
- }
553
-
554
- const row = targets[i];
555
- const hasVoice = row.script.trim().length > 0;
556
-
557
- set((s) => ({
558
- progress: { current: i + 1, total: targets.length },
559
- rows: s.rows.map((r) =>
560
- r.id === row.id
561
- ? {
562
- ...r,
563
- status: hasVoice ? "tts" : "video",
564
- error: null,
565
- logs: [],
566
- result: null,
567
- audioResult: null,
568
- motionResult: null,
569
- }
570
- : r,
571
- ),
572
- logs: [
573
- ...s.logs,
574
- `[${i + 1}/${targets.length}] ${hasVoice ? "Voice" : "Video (no voice)"}: ${row.prompt.trim().slice(0, 80)}`,
575
- ],
576
- }));
577
-
578
- try {
579
- // ===== STEP 1: TTS (skipped when the row has no script) =====
580
- if (hasVoice) {
581
- const ttsRes = await fetch(`${API_BASE}/api/render/tts`, {
582
- method: "POST",
583
- headers: { "Content-Type": "application/json" },
584
- body: JSON.stringify({
585
- text: row.script.trim(),
586
- refAudioPath,
587
- projectId,
588
- quality,
589
- // Each row's voiceover is saved under
590
- // <projectOutputDir>/voices/<voiceId>/, keyed by the row id.
591
- voiceId: row.id,
592
- }),
593
- signal,
594
- });
595
-
596
- if (!ttsRes.ok) {
597
- const err = await ttsRes.text();
598
- set((s) => ({
599
- rows: s.rows.map((r) =>
600
- r.id === row.id ? { ...r, status: "error", error: err } : r,
601
- ),
602
- }));
603
- continue;
604
- }
605
-
606
- let ttsDone = false;
607
- await readSSEStream(ttsRes, (event, data) => {
608
- if (ttsDone) return;
609
- switch (event) {
610
- case "log":
611
- set((s) => ({
612
- rows: s.rows.map((r) =>
613
- r.id === row.id
614
- ? { ...r, logs: [...r.logs, data.text as string] }
615
- : r,
616
- ),
617
- logs: [
618
- ...s.logs,
619
- `[${i + 1}/${targets.length}] ${data.text as string}`,
620
- ],
621
- }));
622
- break;
623
- case "complete":
624
- ttsDone = true;
625
- set((s) => ({
626
- rows: s.rows.map((r) =>
627
- r.id === row.id
628
- ? {
629
- ...r,
630
- audioResult: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
631
- }
632
- : r,
633
- ),
634
- }));
635
- break;
636
- case "error":
637
- ttsDone = true;
638
- set((s) => ({
639
- rows: s.rows.map((r) =>
640
- r.id === row.id
641
- ? {
642
- ...r,
643
- status: "error",
644
- error: data.error || "Voice generation failed",
645
- }
646
- : r,
647
- ),
648
- }));
649
- break;
650
- }
651
- });
652
-
653
- const ttsRow = get().rows.find((r) => r.id === row.id);
654
- if (!ttsRow || ttsRow.status === "error") {
655
- continue;
656
- }
657
- }
658
-
659
- // ===== STEP 2: Video =====
660
- set((s) => ({
661
- rows: s.rows.map((r) =>
662
- r.id === row.id ? { ...r, status: "video" } : r,
663
- ),
664
- }));
665
-
666
- const imagePath = normalizeImagePath(row.imagePath!);
667
-
668
- const vidRes = await fetch(`${API_BASE}/api/render/image-to-video`, {
669
- method: "POST",
670
- headers: { "Content-Type": "application/json" },
671
- body: JSON.stringify({
672
- prompt: row.prompt.trim(),
673
- imagePath,
674
- projectId,
675
- width,
676
- height,
677
- frames: duration * 24 + 1,
678
- frameRate: 24,
679
- mode,
680
- }),
681
- signal,
682
- });
683
-
684
- if (!vidRes.ok) {
685
- const err = await vidRes.text();
686
- set((s) => ({
687
- rows: s.rows.map((r) =>
688
- r.id === row.id ? { ...r, status: "error", error: err } : r,
689
- ),
690
- }));
691
- continue;
692
- }
693
-
694
- let vidDone = false;
695
- await readSSEStream(vidRes, (event, data) => {
696
- if (vidDone) return;
697
- switch (event) {
698
- case "log":
699
- set((s) => ({
700
- rows: s.rows.map((r) =>
701
- r.id === row.id
702
- ? { ...r, logs: [...r.logs, data.text as string] }
703
- : r,
704
- ),
705
- logs: [
706
- ...s.logs,
707
- `[${i + 1}/${targets.length}] ${data.text as string}`,
708
- ],
709
- }));
710
- break;
711
- case "complete":
712
- vidDone = true;
713
- set((s) => ({
714
- rows: s.rows.map((r) =>
715
- r.id === row.id
716
- ? {
717
- ...r,
718
- motionResult: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
719
- }
720
- : r,
721
- ),
722
- }));
723
- break;
724
- case "error":
725
- vidDone = true;
726
- set((s) => ({
727
- rows: s.rows.map((r) =>
728
- r.id === row.id
729
- ? {
730
- ...r,
731
- status: "error",
732
- error: data.error || "Video generation failed",
733
- }
734
- : r,
735
- ),
736
- }));
737
- break;
738
- }
739
- });
740
-
741
- const vidRow = get().rows.find((r) => r.id === row.id);
742
- if (!vidRow || vidRow.status === "error") {
743
- continue;
744
- }
745
-
746
- // No script → no voiceover: the motion video is the final result.
747
- if (!hasVoice) {
748
- set((s) => ({
749
- rows: s.rows.map((r) =>
750
- r.id === row.id
751
- ? { ...r, status: "done", result: vidRow.motionResult }
752
- : r,
753
- ),
754
- }));
755
- continue;
756
- }
757
-
758
- // ===== STEP 3: Mux audio + video =====
759
- set((s) => ({
760
- rows: s.rows.map((r) =>
761
- r.id === row.id ? { ...r, status: "muxing" } : r,
762
- ),
763
- }));
764
-
765
- const muxRes = await fetch(`${API_BASE}/api/render/mux-audio`, {
766
- method: "POST",
767
- headers: { "Content-Type": "application/json" },
768
- body: JSON.stringify({
769
- videoPath: normalizeImagePath(vidRow.motionResult!),
770
- audioPath: normalizeImagePath(vidRow.audioResult!),
771
- projectId,
772
- }),
773
- signal,
774
- });
775
-
776
- if (!muxRes.ok) {
777
- const err = await muxRes.text();
778
- set((s) => ({
779
- rows: s.rows.map((r) =>
780
- r.id === row.id ? { ...r, status: "error", error: err } : r,
781
- ),
782
- }));
783
- continue;
784
- }
785
-
786
- let muxDone = false;
787
- await readSSEStream(muxRes, (event, data) => {
788
- if (muxDone) return;
789
- switch (event) {
790
- case "log":
791
- set((s) => ({
792
- rows: s.rows.map((r) =>
793
- r.id === row.id
794
- ? { ...r, logs: [...r.logs, data.text as string] }
795
- : r,
796
- ),
797
- logs: [
798
- ...s.logs,
799
- `[${i + 1}/${targets.length}] ${data.text as string}`,
800
- ],
801
- }));
802
- break;
803
- case "complete":
804
- muxDone = true;
805
- set((s) => ({
806
- rows: s.rows.map((r) =>
807
- r.id === row.id
808
- ? {
809
- ...r,
810
- status: "done",
811
- result: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
812
- }
813
- : r,
814
- ),
815
- }));
816
- break;
817
- case "error":
818
- muxDone = true;
819
- set((s) => ({
820
- rows: s.rows.map((r) =>
821
- r.id === row.id
822
- ? {
823
- ...r,
824
- status: "error",
825
- error: data.error || "Muxing failed",
826
- }
827
- : r,
828
- ),
829
- }));
830
- break;
831
- }
832
- });
833
- } catch (e: any) {
834
- if (e?.name === "AbortError") {
835
- cancelled = true;
836
- break;
837
- }
838
- set((s) => ({
839
- rows: s.rows.map((r) =>
840
- r.id === row.id
841
- ? { ...r, status: "error", error: String(e) }
842
- : r,
843
- ),
844
- }));
845
- }
846
- }
847
-
848
- batchAbortController = null;
849
- set({ running: false, progress: null, cancelRequested: false });
850
-
851
- if (!cancelled) {
852
- playBeep();
853
- }
854
- },
855
-
856
- generateRow: (projectId, id) => get().generateRows(projectId, [id]),
857
-
858
- generateAll: (projectId) => {
859
- const ids = get()
860
- .rows.filter(
861
- (r) =>
862
- r.prompt.trim() &&
863
- r.imagePath &&
864
- (!r.script.trim() || get().voiceRefPath),
865
- )
866
- .map((r) => r.id);
867
- return get().generateRows(projectId, ids);
868
- },
869
-
870
- cancel: () => {
871
- set({ cancelRequested: true });
872
- if (batchAbortController) {
873
- batchAbortController.abort();
874
- }
875
- set({
876
- running: false,
877
- progress: null,
878
- rows: get().rows.map((r) =>
879
- r.status === "tts" || r.status === "video" || r.status === "muxing"
880
- ? { ...r, status: "idle" }
881
- : r,
882
- ),
883
- });
884
- // Keep cancelRequested: true — the loop's top-of-iteration check breaks out
885
- // between rows, and generateRows resets the flag in its final set().
886
- fetch(`${API_BASE}/api/render/cancel`, { method: "POST" }).catch(() => {});
887
- },
888
-
889
- stitchVideos: async () => {
890
- if (get().stitching) return;
891
-
892
- const results = get()
893
- .rows.filter((r) => r.result)
894
- .map((r) => r.result as string);
895
-
896
- if (results.length < 2) {
897
- set({
898
- stitchError: "Need at least 2 generated videos to stitch.",
899
- stitchResult: null,
900
- });
901
- return;
902
- }
903
-
904
- set({
905
- stitching: true,
906
- stitchLogs: [],
907
- stitchResult: null,
908
- stitchError: null,
909
- });
910
-
911
- try {
912
- const ffmpeg = await loadFFmpeg();
913
-
914
- // Write each generated video into the in-memory FS in order.
915
- const list: string[] = [];
916
- for (let i = 0; i < results.length; i++) {
917
- const blob = await fetch(results[i]).then((r) => {
918
- if (!r.ok) throw new Error(`Failed to fetch video ${i + 1}`);
919
- return r.blob();
920
- });
921
- const data = new Uint8Array(await blob.arrayBuffer());
922
- const name = `in${i}.mp4`;
923
- await ffmpeg.writeFile(name, data);
924
- list.push(`file '${name}'`);
925
- set((s) => ({
926
- stitchLogs: [...s.stitchLogs, `Added video ${i + 1}/${results.length}`],
927
- }));
928
- }
929
-
930
- await ffmpeg.writeFile("list.txt", list.join("\n"));
931
- set((s) => ({
932
- stitchLogs: [...s.stitchLogs, "Concatenating videos..."],
933
- }));
934
-
935
- const ret = await ffmpeg.exec([
936
- "-f",
937
- "concat",
938
- "-safe",
939
- "0",
940
- "-i",
941
- "list.txt",
942
- "-c",
943
- "copy",
944
- "output.mp4",
945
- ]);
946
-
947
- if (ret !== 0) {
948
- throw new Error(`ffmpeg exited with code ${ret}`);
949
- }
950
-
951
- const out = (await ffmpeg.readFile("output.mp4")) as Uint8Array;
952
- const bytes = new Uint8Array(out);
953
- const url = URL.createObjectURL(
954
- new Blob([bytes.buffer], { type: "video/mp4" }),
955
- );
956
-
957
- set({
958
- stitching: false,
959
- stitchResult: url,
960
- stitchLogs: [...get().stitchLogs, "Done"],
961
- });
962
- } catch (e) {
963
- set({
964
- stitching: false,
965
- stitchError: String(e),
966
- });
967
- }
968
- },
969
-
970
- reset: () =>
971
- set({
972
- rows: [makeRow()],
973
- duration: 5,
974
- aspectRatio: "1:1",
975
- resolution: "480p",
976
- mode: "distilled",
977
- quality: "high",
978
- voiceRefPath: null,
979
- voiceRefUrl: null,
980
- voiceRefFilename: null,
981
- running: false,
982
- progress: null,
983
- cancelRequested: false,
984
- logs: [],
985
- stitching: false,
986
- stitchLogs: [],
987
- stitchResult: null,
988
- stitchError: null,
989
- }),
990
- }));