@effectnode/media 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/bin/effectnode-media +2 -0
  2. package/dist/backend/index.d.ts +8 -0
  3. package/dist/backend/index.js +13 -0
  4. package/dist/backend/movie-backend/agent/agent-backend.d.ts +16 -0
  5. package/dist/backend/movie-backend/agent/agent-backend.js +703 -0
  6. package/dist/backend/movie-backend/agent/storyprompt.d.ts +1 -0
  7. package/dist/backend/movie-backend/agent/storyprompt.js +90 -0
  8. package/dist/backend/movie-backend/agent/tools/edit-image.d.ts +3 -0
  9. package/dist/backend/movie-backend/agent/tools/edit-image.js +192 -0
  10. package/dist/backend/movie-backend/agent/tools/get-time.d.ts +3 -0
  11. package/dist/backend/movie-backend/agent/tools/get-time.js +7 -0
  12. package/dist/backend/movie-backend/agent/tools/grep-files.d.ts +3 -0
  13. package/dist/backend/movie-backend/agent/tools/grep-files.js +59 -0
  14. package/dist/backend/movie-backend/agent/tools/image-to-video-generation.d.ts +3 -0
  15. package/dist/backend/movie-backend/agent/tools/image-to-video-generation.js +160 -0
  16. package/dist/backend/movie-backend/agent/tools/index.d.ts +7 -0
  17. package/dist/backend/movie-backend/agent/tools/index.js +59 -0
  18. package/dist/backend/movie-backend/agent/tools/list-files.d.ts +3 -0
  19. package/dist/backend/movie-backend/agent/tools/list-files.js +17 -0
  20. package/dist/backend/movie-backend/agent/tools/read-file.d.ts +3 -0
  21. package/dist/backend/movie-backend/agent/tools/read-file.js +28 -0
  22. package/dist/backend/movie-backend/agent/tools/remove-file.d.ts +3 -0
  23. package/dist/backend/movie-backend/agent/tools/remove-file.js +22 -0
  24. package/dist/backend/movie-backend/agent/tools/rename-file.d.ts +3 -0
  25. package/dist/backend/movie-backend/agent/tools/rename-file.js +37 -0
  26. package/dist/backend/movie-backend/agent/tools/show-image.d.ts +3 -0
  27. package/dist/backend/movie-backend/agent/tools/show-image.js +28 -0
  28. package/dist/backend/movie-backend/agent/tools/stitch-videos.d.ts +3 -0
  29. package/dist/backend/movie-backend/agent/tools/stitch-videos.js +96 -0
  30. package/dist/backend/movie-backend/agent/tools/text-to-video-generation.d.ts +3 -0
  31. package/dist/backend/movie-backend/agent/tools/text-to-video-generation.js +125 -0
  32. package/dist/backend/movie-backend/agent/tools/types.d.ts +15 -0
  33. package/dist/backend/movie-backend/agent/tools/types.js +1 -0
  34. package/dist/backend/movie-backend/agent/tools/update-file.d.ts +3 -0
  35. package/dist/backend/movie-backend/agent/tools/update-file.js +27 -0
  36. package/dist/backend/movie-backend/agent/tools/write-file.d.ts +3 -0
  37. package/dist/backend/movie-backend/agent/tools/write-file.js +26 -0
  38. package/dist/backend/movie-backend/agent/workspace.d.ts +25 -0
  39. package/dist/backend/movie-backend/agent/workspace.js +124 -0
  40. package/dist/backend/movie-backend/core.d.ts +4 -0
  41. package/dist/backend/movie-backend/core.js +1167 -0
  42. package/dist/backend/movie-backend/generation-queue.d.ts +24 -0
  43. package/dist/backend/movie-backend/generation-queue.js +766 -0
  44. package/dist/backend/movie-backend/process.d.ts +21 -0
  45. package/dist/backend/movie-backend/process.js +98 -0
  46. package/dist/backend/movie-backend/render-media.d.ts +48 -0
  47. package/dist/backend/movie-backend/render-media.js +3311 -0
  48. package/dist/backend/routes/health.d.ts +1 -0
  49. package/dist/backend/routes/health.js +9 -0
  50. package/dist/backend/routes/index.d.ts +1 -0
  51. package/dist/backend/routes/index.js +11 -0
  52. package/dist/backend/routes/media.d.ts +1 -0
  53. package/dist/backend/routes/media.js +13 -0
  54. package/dist/backend/routes/projects.d.ts +8 -0
  55. package/dist/backend/routes/projects.js +68 -0
  56. package/dist/backend/routes/voice.d.ts +1 -0
  57. package/dist/backend/routes/voice.js +39 -0
  58. package/dist/backend/routes.d.ts +1 -0
  59. package/dist/backend/routes.js +20 -0
  60. package/dist/backend/server.d.ts +1 -0
  61. package/dist/backend/server.js +23 -0
  62. package/dist/backend/workspace.d.ts +5 -0
  63. package/dist/backend/workspace.js +27 -0
  64. package/dist/backend/ws/index.d.ts +3 -0
  65. package/dist/backend/ws/index.js +27 -0
  66. package/dist/backend/ws/messages.d.ts +8 -0
  67. package/dist/backend/ws/messages.js +21 -0
  68. package/dist/backend/ws.d.ts +3 -0
  69. package/dist/backend/ws.js +29 -0
  70. package/dist/index.d.ts +2 -0
  71. package/dist/index.js +80 -0
  72. package/frontend/index.html +22 -0
  73. package/frontend/public/ffmpeg/ffmpeg-core.js +16 -0
  74. package/frontend/public/ffmpeg/ffmpeg-core.wasm +0 -0
  75. package/frontend/src/index.css +46 -0
  76. package/frontend/src/main.tsx +13 -0
  77. package/frontend/src/movie-app/AppRouter.tsx +14 -0
  78. package/frontend/src/movie-app/MediaStudio.tsx +9 -0
  79. package/frontend/src/movie-app/SetupPage.tsx +293 -0
  80. package/frontend/src/movie-app/components/AgentWorkspace.tsx +574 -0
  81. package/frontend/src/movie-app/components/ChatUI/ChatUI.tsx +655 -0
  82. package/frontend/src/movie-app/components/ChatUI/MarkdownMessage.tsx +267 -0
  83. package/frontend/src/movie-app/components/EditorTabs/AgentTab.tsx +61 -0
  84. package/frontend/src/movie-app/components/EditorTabs/BatchImageToVideoTab.tsx +927 -0
  85. package/frontend/src/movie-app/components/EditorTabs/BatchVideoTab.tsx +777 -0
  86. package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +913 -0
  87. package/frontend/src/movie-app/components/EditorTabs/CharacterSheet.tsx +234 -0
  88. package/frontend/src/movie-app/components/EditorTabs/CharactersTab.tsx +664 -0
  89. package/frontend/src/movie-app/components/EditorTabs/CropTool.tsx +251 -0
  90. package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +305 -0
  91. package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +249 -0
  92. package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +555 -0
  93. package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +722 -0
  94. package/frontend/src/movie-app/components/EditorTabs/LlmServerTab.tsx +40 -0
  95. package/frontend/src/movie-app/components/EditorTabs/MlxVlmServerPanel.tsx +490 -0
  96. package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +938 -0
  97. package/frontend/src/movie-app/components/EditorTabs/ReferencesToVideoTab.tsx +881 -0
  98. package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +267 -0
  99. package/frontend/src/movie-app/components/EditorTabs/StoryWriterTab.tsx +51 -0
  100. package/frontend/src/movie-app/components/EditorTabs/TaskQueuePanel.tsx +246 -0
  101. package/frontend/src/movie-app/components/EditorTabs/TerminalLogPanel.tsx +49 -0
  102. package/frontend/src/movie-app/components/EditorTabs/TextToImageTab.tsx +574 -0
  103. package/frontend/src/movie-app/components/EditorTabs/VoiceChatPanel.tsx +320 -0
  104. package/frontend/src/movie-app/components/ProjectEditorPage.tsx +636 -0
  105. package/frontend/src/movie-app/components/ProjectManager.tsx +381 -0
  106. package/frontend/src/movie-app/index.css +46 -0
  107. package/frontend/src/movie-app/index.html +16 -0
  108. package/frontend/src/movie-app/lambobo.png +0 -0
  109. package/frontend/src/movie-app/lib/batchImageToVideoStorage.ts +69 -0
  110. package/frontend/src/movie-app/lib/batchVideoStorage.ts +68 -0
  111. package/frontend/src/movie-app/lib/batchVoiceStorage.ts +75 -0
  112. package/frontend/src/movie-app/lib/ffmpeg.ts +42 -0
  113. package/frontend/src/movie-app/lib/referencesToVideoStorage.ts +64 -0
  114. package/frontend/src/movie-app/lib/textToImageStorage.ts +61 -0
  115. package/frontend/src/movie-app/public/ffmpeg/ffmpeg-core.js +16 -0
  116. package/frontend/src/movie-app/public/ffmpeg/ffmpeg-core.wasm +0 -0
  117. package/frontend/src/movie-app/stores/batchImageToVideoStore.ts +736 -0
  118. package/frontend/src/movie-app/stores/batchVideoStore.ts +670 -0
  119. package/frontend/src/movie-app/stores/batchVoiceStore.ts +990 -0
  120. package/frontend/src/movie-app/stores/characterStore.ts +109 -0
  121. package/frontend/src/movie-app/stores/chatStore.ts +459 -0
  122. package/frontend/src/movie-app/stores/generationStore.ts +2140 -0
  123. package/frontend/src/movie-app/stores/logStore.ts +25 -0
  124. package/frontend/src/movie-app/stores/movieStudioStore.ts +709 -0
  125. package/frontend/src/movie-app/stores/projectStore.ts +124 -0
  126. package/frontend/src/movie-app/stores/queueStore.ts +214 -0
  127. package/frontend/src/movie-app/stores/referencesToVideoStore.ts +467 -0
  128. package/frontend/src/movie-app/stores/sceneVisualStore.ts +251 -0
  129. package/frontend/src/movie-app/stores/voiceChatStore.ts +170 -0
  130. package/frontend/src/movie-app/stores/workspaceStore.ts +158 -0
  131. package/frontend/src/vite-env.d.ts +1 -0
  132. package/frontend/tsconfig.json +19 -0
  133. package/package.json +65 -0
@@ -0,0 +1,990 @@
1
+ import { create } from "zustand";
2
+ import type {
3
+ AspectRatio,
4
+ Resolution,
5
+ VideoMode,
6
+ ProjectImage,
7
+ } from "./generationStore";
8
+ import { loadFFmpeg } from "../lib/ffmpeg";
9
+ import {
10
+ clearBatchVoiceState,
11
+ loadBatchVoiceState,
12
+ saveBatchVoiceState,
13
+ type PersistedBatchVoiceState,
14
+ type VoiceQuality,
15
+ } from "../lib/batchVoiceStorage";
16
+
17
+ const API_BASE = `http://localhost:${(window as any).PORT}`;
18
+
19
+ export type BatchVoiceRowStatus =
20
+ | "idle"
21
+ | "uploading"
22
+ | "tts"
23
+ | "video"
24
+ | "muxing"
25
+ | "done"
26
+ | "error";
27
+
28
+ export interface BatchVoiceRow {
29
+ id: string;
30
+ // Video prompt (drives the image-to-video stage)
31
+ prompt: string;
32
+ // TTS text (spoken in the cloned voice). Empty = silent video (no voiceover).
33
+ script: string;
34
+ imagePath: string | null;
35
+ imageUrl: string | null;
36
+ imageFilename: string | null;
37
+ status: BatchVoiceRowStatus;
38
+ result: string | null;
39
+ error: string | null;
40
+ logs: string[];
41
+ audioResult: string | null;
42
+ motionResult: string | null;
43
+ }
44
+
45
+ interface BatchVoiceStore {
46
+ // Rows
47
+ rows: BatchVoiceRow[];
48
+
49
+ // Shared video settings
50
+ duration: number;
51
+ aspectRatio: AspectRatio;
52
+ resolution: Resolution;
53
+ mode: VideoMode;
54
+
55
+ // Shared voice settings
56
+ quality: VoiceQuality;
57
+ voiceRefPath: string | null;
58
+ voiceRefUrl: string | null;
59
+ voiceRefFilename: string | null;
60
+
61
+ // Batch generation state
62
+ running: boolean;
63
+ progress: { current: number; total: number } | null;
64
+ cancelRequested: boolean;
65
+ logs: string[];
66
+
67
+ // Stitching state
68
+ stitching: boolean;
69
+ stitchLogs: string[];
70
+ stitchResult: string | null;
71
+ stitchError: string | null;
72
+
73
+ // Persistence
74
+ projectId: string | null;
75
+ hydrated: boolean;
76
+ hydrate: (projectId: string) => Promise<void>;
77
+ clear: () => void;
78
+
79
+ addRow: () => void;
80
+ removeRow: (id: string) => void;
81
+ updatePrompt: (id: string, prompt: string) => void;
82
+ updateScript: (id: string, script: string) => void;
83
+ clearRowResult: (id: string) => void;
84
+ uploadRowImage: (
85
+ id: string,
86
+ base64: string,
87
+ filename: string | undefined,
88
+ projectId: string,
89
+ ) => Promise<string | null>;
90
+ setRowImage: (id: string, image: ProjectImage) => void;
91
+
92
+ setDuration: (v: number) => void;
93
+ setAspectRatio: (v: AspectRatio) => void;
94
+ setResolution: (v: Resolution) => void;
95
+ setMode: (v: VideoMode) => void;
96
+ setQuality: (v: VoiceQuality) => void;
97
+ uploadVoiceRef: (
98
+ base64: string,
99
+ filename: string,
100
+ projectId: string,
101
+ ) => Promise<string | null>;
102
+
103
+ generateRows: (projectId: string, ids: string[]) => Promise<void>;
104
+ generateRow: (projectId: string, id: string) => Promise<void>;
105
+ generateAll: (projectId: string) => Promise<void>;
106
+ cancel: () => void;
107
+
108
+ stitchVideos: () => Promise<void>;
109
+
110
+ reset: () => void;
111
+ }
112
+
113
+ let batchAbortController: AbortController | null = null;
114
+
115
+ function getDimensions(
116
+ aspect: AspectRatio,
117
+ resolution: Resolution,
118
+ ): { width: number; height: number } {
119
+ const size = parseInt(resolution);
120
+ switch (aspect) {
121
+ case "1:1":
122
+ return { width: size, height: size };
123
+ case "16:9":
124
+ return { width: Math.round((size * 16) / 9), height: size };
125
+ case "9:16":
126
+ return { width: size, height: Math.round((size * 16) / 9) };
127
+ case "4:3":
128
+ return { width: Math.round((size * 4) / 3), height: size };
129
+ case "3:4":
130
+ return { width: size, height: Math.round((size * 4) / 3) };
131
+ }
132
+ }
133
+
134
+ // The image-to-video backend only accepts a bare filename, so normalise any
135
+ // uploaded path / file URL / api-files URL down to just the basename.
136
+ function normalizeImagePath(path: string): string {
137
+ let p = path;
138
+ if (p.includes("/api/files?path=")) {
139
+ try {
140
+ const url = new URL(p);
141
+ p = url.searchParams.get("path") || p;
142
+ } catch {
143
+ // not a valid URL, use as-is
144
+ }
145
+ }
146
+ if (p.startsWith("file://")) {
147
+ p = p.slice(7);
148
+ }
149
+ return p.split("/").pop() || p;
150
+ }
151
+
152
+ async function readSSEStream(
153
+ response: Response,
154
+ onEvent: (event: string, data: any) => void,
155
+ ): Promise<void> {
156
+ const reader = response.body?.getReader();
157
+ if (!reader) return;
158
+
159
+ const decoder = new TextDecoder();
160
+ let buffer = "";
161
+
162
+ try {
163
+ while (true) {
164
+ const { done, value } = await reader.read();
165
+ if (done) break;
166
+
167
+ buffer += decoder.decode(value, { stream: true });
168
+ const lines = buffer.split("\n");
169
+ buffer = lines.pop() || "";
170
+
171
+ let eventType = "message";
172
+ for (const line of lines) {
173
+ if (line.startsWith("event: ")) {
174
+ eventType = line.slice(7).trim();
175
+ } else if (line.startsWith("data: ")) {
176
+ try {
177
+ onEvent(eventType, JSON.parse(line.slice(6)));
178
+ } catch {
179
+ // skip malformed lines
180
+ }
181
+ eventType = "message";
182
+ }
183
+ }
184
+ }
185
+ } finally {
186
+ reader.releaseLock();
187
+ }
188
+ }
189
+
190
+ function playBeep() {
191
+ try {
192
+ const ctx = new (
193
+ window.AudioContext || (window as any).webkitAudioContext
194
+ )();
195
+ const osc = ctx.createOscillator();
196
+ const gain = ctx.createGain();
197
+ osc.connect(gain);
198
+ gain.connect(ctx.destination);
199
+ osc.type = "sine";
200
+ osc.frequency.setValueAtTime(880, ctx.currentTime);
201
+ gain.gain.setValueAtTime(0.3, ctx.currentTime);
202
+ gain.gain.exponentialRampToValueAtTime(0.01, ctx.currentTime + 0.4);
203
+ osc.start(ctx.currentTime);
204
+ osc.stop(ctx.currentTime + 0.4);
205
+ } catch {
206
+ // silently ignore if audio not available
207
+ }
208
+ }
209
+
210
+ function makeRow(): BatchVoiceRow {
211
+ return {
212
+ id: `row-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`,
213
+ prompt: "",
214
+ script: "",
215
+ imagePath: null,
216
+ imageUrl: null,
217
+ imageFilename: null,
218
+ status: "idle",
219
+ result: null,
220
+ error: null,
221
+ logs: [],
222
+ audioResult: null,
223
+ motionResult: null,
224
+ };
225
+ }
226
+
227
+ function toPersistedState(
228
+ s: Pick<
229
+ BatchVoiceStore,
230
+ | "rows"
231
+ | "duration"
232
+ | "aspectRatio"
233
+ | "resolution"
234
+ | "mode"
235
+ | "quality"
236
+ | "voiceRefPath"
237
+ | "voiceRefFilename"
238
+ >,
239
+ ): PersistedBatchVoiceState {
240
+ return {
241
+ rows: s.rows.map((r) => ({
242
+ id: r.id,
243
+ prompt: r.prompt,
244
+ script: r.script,
245
+ imagePath: r.imagePath,
246
+ imageUrl: r.imageUrl,
247
+ imageFilename: r.imageFilename,
248
+ })),
249
+ duration: s.duration,
250
+ aspectRatio: s.aspectRatio,
251
+ resolution: s.resolution,
252
+ mode: s.mode,
253
+ quality: s.quality,
254
+ voiceRefPath: s.voiceRefPath,
255
+ voiceRefFilename: s.voiceRefFilename,
256
+ };
257
+ }
258
+
259
+ // Fire-and-forget save of the current editable UI state.
260
+ function persistBatchState() {
261
+ const { projectId } = useBatchVoiceStore.getState();
262
+ if (!projectId) return;
263
+ void saveBatchVoiceState(
264
+ projectId,
265
+ toPersistedState(useBatchVoiceStore.getState()),
266
+ );
267
+ }
268
+
269
+ export const useBatchVoiceStore = create<BatchVoiceStore>((set, get) => ({
270
+ rows: [makeRow()],
271
+
272
+ duration: 5,
273
+ aspectRatio: "1:1",
274
+ resolution: "480p",
275
+ mode: "distilled",
276
+
277
+ quality: "high",
278
+ voiceRefPath: null,
279
+ voiceRefUrl: null,
280
+ voiceRefFilename: null,
281
+
282
+ running: false,
283
+ progress: null,
284
+ cancelRequested: false,
285
+ logs: [],
286
+
287
+ stitching: false,
288
+ stitchLogs: [],
289
+ stitchResult: null,
290
+ stitchError: null,
291
+
292
+ projectId: null,
293
+ hydrated: false,
294
+
295
+ hydrate: async (projectId) => {
296
+ // No-op if we've already hydrated for this project.
297
+ if (get().hydrated && get().projectId === projectId) return;
298
+
299
+ // Switching to a different project: reset to defaults so rows and
300
+ // settings from the previous project don't leak through.
301
+ const previous = get().projectId;
302
+ if (previous !== null && previous !== projectId) {
303
+ get().reset();
304
+ }
305
+ set({ hydrated: true, projectId });
306
+
307
+ const stored = await loadBatchVoiceState(projectId);
308
+ if (!stored) return;
309
+
310
+ set((s) => {
311
+ const rows: BatchVoiceRow[] = (stored.rows ?? []).map((r) => ({
312
+ id: r.id,
313
+ prompt: r.prompt,
314
+ script: r.script,
315
+ imagePath: r.imagePath,
316
+ imageUrl: r.imageUrl,
317
+ imageFilename: r.imageFilename,
318
+ status: "idle",
319
+ result: null,
320
+ error: null,
321
+ logs: [],
322
+ audioResult: null,
323
+ motionResult: null,
324
+ }));
325
+
326
+ // Rebuild the preview URL for the stored voice reference so the audio
327
+ // player keeps working after a reload.
328
+ const restoredRefPath = stored.voiceRefPath ?? s.voiceRefPath;
329
+
330
+ return {
331
+ rows: rows.length > 0 ? rows : [makeRow()],
332
+ duration: stored.duration ?? s.duration,
333
+ aspectRatio: stored.aspectRatio ?? s.aspectRatio,
334
+ resolution: stored.resolution ?? s.resolution,
335
+ mode: stored.mode ?? s.mode,
336
+ quality: stored.quality ?? s.quality,
337
+ voiceRefPath: restoredRefPath,
338
+ voiceRefUrl: restoredRefPath
339
+ ? `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(restoredRefPath)}`
340
+ : null,
341
+ voiceRefFilename: stored.voiceRefFilename ?? s.voiceRefFilename,
342
+ };
343
+ });
344
+ },
345
+
346
+ clear: () => {
347
+ const { projectId } = get();
348
+ get().reset();
349
+ if (projectId) void clearBatchVoiceState(projectId);
350
+ },
351
+
352
+ addRow: () => {
353
+ set((s) => ({ rows: [...s.rows, makeRow()] }));
354
+ persistBatchState();
355
+ },
356
+
357
+ removeRow: (id) => {
358
+ set((s) => ({ rows: s.rows.filter((r) => r.id !== id) }));
359
+ persistBatchState();
360
+ },
361
+
362
+ updatePrompt: (id, prompt) => {
363
+ set((s) => ({
364
+ rows: s.rows.map((r) => (r.id === id ? { ...r, prompt } : r)),
365
+ }));
366
+ persistBatchState();
367
+ },
368
+
369
+ updateScript: (id, script) => {
370
+ set((s) => ({
371
+ rows: s.rows.map((r) => (r.id === id ? { ...r, script } : r)),
372
+ }));
373
+ persistBatchState();
374
+ },
375
+
376
+ clearRowResult: (id) =>
377
+ set((s) => ({
378
+ rows: s.rows.map((r) =>
379
+ r.id === id
380
+ ? {
381
+ ...r,
382
+ result: null,
383
+ error: null,
384
+ status: "idle",
385
+ audioResult: null,
386
+ motionResult: null,
387
+ }
388
+ : r,
389
+ ),
390
+ })),
391
+
392
+ uploadRowImage: async (id, base64, filename, projectId) => {
393
+ set((s) => ({
394
+ rows: s.rows.map((r) =>
395
+ r.id === id ? { ...r, status: "uploading" } : r,
396
+ ),
397
+ }));
398
+
399
+ try {
400
+ const res = await fetch(`${API_BASE}/api/upload/image`, {
401
+ method: "POST",
402
+ headers: { "Content-Type": "application/json" },
403
+ body: JSON.stringify({
404
+ image: base64,
405
+ filename: filename || `upload-${Date.now()}.png`,
406
+ projectId,
407
+ }),
408
+ });
409
+
410
+ if (!res.ok) {
411
+ const err = await res.text();
412
+ set((s) => ({
413
+ rows: s.rows.map((r) =>
414
+ r.id === id ? { ...r, status: "error", error: err } : r,
415
+ ),
416
+ }));
417
+ return null;
418
+ }
419
+
420
+ const data = await res.json();
421
+ const url = `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`;
422
+ set((s) => ({
423
+ rows: s.rows.map((r) =>
424
+ r.id === id
425
+ ? {
426
+ ...r,
427
+ status: "idle",
428
+ imagePath: data.path,
429
+ imageUrl: url,
430
+ imageFilename: data.filename,
431
+ }
432
+ : r,
433
+ ),
434
+ }));
435
+ persistBatchState();
436
+ return data.path as string;
437
+ } catch (e) {
438
+ set((s) => ({
439
+ rows: s.rows.map((r) =>
440
+ r.id === id ? { ...r, status: "error", error: String(e) } : r,
441
+ ),
442
+ }));
443
+ return null;
444
+ }
445
+ },
446
+
447
+ setRowImage: (id, image) => {
448
+ set((s) => ({
449
+ rows: s.rows.map((r) =>
450
+ r.id === id
451
+ ? {
452
+ ...r,
453
+ imagePath: image.url,
454
+ imageUrl: image.url,
455
+ imageFilename: image.filename,
456
+ status: "idle",
457
+ error: null,
458
+ }
459
+ : r,
460
+ ),
461
+ }));
462
+ persistBatchState();
463
+ },
464
+
465
+ setDuration: (duration) => {
466
+ set({ duration });
467
+ persistBatchState();
468
+ },
469
+ setAspectRatio: (aspectRatio) => {
470
+ set({ aspectRatio });
471
+ persistBatchState();
472
+ },
473
+ setResolution: (resolution) => {
474
+ set({ resolution });
475
+ persistBatchState();
476
+ },
477
+ setMode: (mode) => {
478
+ set({ mode });
479
+ persistBatchState();
480
+ },
481
+ setQuality: (quality) => {
482
+ set({ quality });
483
+ persistBatchState();
484
+ },
485
+
486
+ uploadVoiceRef: async (base64, filename, projectId) => {
487
+ try {
488
+ const res = await fetch(`${API_BASE}/api/upload/audio`, {
489
+ method: "POST",
490
+ headers: { "Content-Type": "application/json" },
491
+ body: JSON.stringify({
492
+ audio: base64,
493
+ filename,
494
+ projectId,
495
+ }),
496
+ });
497
+
498
+ if (!res.ok) {
499
+ return null;
500
+ }
501
+
502
+ const data = await res.json();
503
+ set({
504
+ voiceRefPath: data.path,
505
+ voiceRefUrl: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
506
+ voiceRefFilename: data.filename,
507
+ });
508
+ persistBatchState();
509
+ return data.path as string;
510
+ } catch {
511
+ return null;
512
+ }
513
+ },
514
+
515
+ generateRows: async (projectId, ids) => {
516
+ if (get().running) return;
517
+
518
+ const rows = get().rows;
519
+ const targets = ids
520
+ .map((id) => rows.find((r) => r.id === id))
521
+ .filter((r): r is BatchVoiceRow => !!r)
522
+ .filter(
523
+ (r) =>
524
+ r.prompt.trim() &&
525
+ r.imagePath &&
526
+ (!r.script.trim() || get().voiceRefPath),
527
+ );
528
+
529
+ if (targets.length === 0) return;
530
+
531
+ const { duration, aspectRatio, resolution, mode, quality, voiceRefPath } =
532
+ get();
533
+ const { width, height } = getDimensions(aspectRatio, resolution);
534
+ const refAudioPath = voiceRefPath ? normalizeImagePath(voiceRefPath) : null;
535
+
536
+ batchAbortController = new AbortController();
537
+ const signal = batchAbortController.signal;
538
+
539
+ set({
540
+ running: true,
541
+ progress: { current: 0, total: targets.length },
542
+ cancelRequested: false,
543
+ logs: [],
544
+ });
545
+
546
+ let cancelled = false;
547
+
548
+ for (let i = 0; i < targets.length; i++) {
549
+ if (get().cancelRequested) {
550
+ cancelled = true;
551
+ break;
552
+ }
553
+
554
+ const row = targets[i];
555
+ const hasVoice = row.script.trim().length > 0;
556
+
557
+ set((s) => ({
558
+ progress: { current: i + 1, total: targets.length },
559
+ rows: s.rows.map((r) =>
560
+ r.id === row.id
561
+ ? {
562
+ ...r,
563
+ status: hasVoice ? "tts" : "video",
564
+ error: null,
565
+ logs: [],
566
+ result: null,
567
+ audioResult: null,
568
+ motionResult: null,
569
+ }
570
+ : r,
571
+ ),
572
+ logs: [
573
+ ...s.logs,
574
+ `[${i + 1}/${targets.length}] ${hasVoice ? "Voice" : "Video (no voice)"}: ${row.prompt.trim().slice(0, 80)}`,
575
+ ],
576
+ }));
577
+
578
+ try {
579
+ // ===== STEP 1: TTS (skipped when the row has no script) =====
580
+ if (hasVoice) {
581
+ const ttsRes = await fetch(`${API_BASE}/api/render/tts`, {
582
+ method: "POST",
583
+ headers: { "Content-Type": "application/json" },
584
+ body: JSON.stringify({
585
+ text: row.script.trim(),
586
+ refAudioPath,
587
+ projectId,
588
+ quality,
589
+ // Each row's voiceover is saved under
590
+ // <projectOutputDir>/voices/<voiceId>/, keyed by the row id.
591
+ voiceId: row.id,
592
+ }),
593
+ signal,
594
+ });
595
+
596
+ if (!ttsRes.ok) {
597
+ const err = await ttsRes.text();
598
+ set((s) => ({
599
+ rows: s.rows.map((r) =>
600
+ r.id === row.id ? { ...r, status: "error", error: err } : r,
601
+ ),
602
+ }));
603
+ continue;
604
+ }
605
+
606
+ let ttsDone = false;
607
+ await readSSEStream(ttsRes, (event, data) => {
608
+ if (ttsDone) return;
609
+ switch (event) {
610
+ case "log":
611
+ set((s) => ({
612
+ rows: s.rows.map((r) =>
613
+ r.id === row.id
614
+ ? { ...r, logs: [...r.logs, data.text as string] }
615
+ : r,
616
+ ),
617
+ logs: [
618
+ ...s.logs,
619
+ `[${i + 1}/${targets.length}] ${data.text as string}`,
620
+ ],
621
+ }));
622
+ break;
623
+ case "complete":
624
+ ttsDone = true;
625
+ set((s) => ({
626
+ rows: s.rows.map((r) =>
627
+ r.id === row.id
628
+ ? {
629
+ ...r,
630
+ audioResult: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
631
+ }
632
+ : r,
633
+ ),
634
+ }));
635
+ break;
636
+ case "error":
637
+ ttsDone = true;
638
+ set((s) => ({
639
+ rows: s.rows.map((r) =>
640
+ r.id === row.id
641
+ ? {
642
+ ...r,
643
+ status: "error",
644
+ error: data.error || "Voice generation failed",
645
+ }
646
+ : r,
647
+ ),
648
+ }));
649
+ break;
650
+ }
651
+ });
652
+
653
+ const ttsRow = get().rows.find((r) => r.id === row.id);
654
+ if (!ttsRow || ttsRow.status === "error") {
655
+ continue;
656
+ }
657
+ }
658
+
659
+ // ===== STEP 2: Video =====
660
+ set((s) => ({
661
+ rows: s.rows.map((r) =>
662
+ r.id === row.id ? { ...r, status: "video" } : r,
663
+ ),
664
+ }));
665
+
666
+ const imagePath = normalizeImagePath(row.imagePath!);
667
+
668
+ const vidRes = await fetch(`${API_BASE}/api/render/image-to-video`, {
669
+ method: "POST",
670
+ headers: { "Content-Type": "application/json" },
671
+ body: JSON.stringify({
672
+ prompt: row.prompt.trim(),
673
+ imagePath,
674
+ projectId,
675
+ width,
676
+ height,
677
+ frames: duration * 24 + 1,
678
+ frameRate: 24,
679
+ mode,
680
+ }),
681
+ signal,
682
+ });
683
+
684
+ if (!vidRes.ok) {
685
+ const err = await vidRes.text();
686
+ set((s) => ({
687
+ rows: s.rows.map((r) =>
688
+ r.id === row.id ? { ...r, status: "error", error: err } : r,
689
+ ),
690
+ }));
691
+ continue;
692
+ }
693
+
694
+ let vidDone = false;
695
+ await readSSEStream(vidRes, (event, data) => {
696
+ if (vidDone) return;
697
+ switch (event) {
698
+ case "log":
699
+ set((s) => ({
700
+ rows: s.rows.map((r) =>
701
+ r.id === row.id
702
+ ? { ...r, logs: [...r.logs, data.text as string] }
703
+ : r,
704
+ ),
705
+ logs: [
706
+ ...s.logs,
707
+ `[${i + 1}/${targets.length}] ${data.text as string}`,
708
+ ],
709
+ }));
710
+ break;
711
+ case "complete":
712
+ vidDone = true;
713
+ set((s) => ({
714
+ rows: s.rows.map((r) =>
715
+ r.id === row.id
716
+ ? {
717
+ ...r,
718
+ motionResult: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
719
+ }
720
+ : r,
721
+ ),
722
+ }));
723
+ break;
724
+ case "error":
725
+ vidDone = true;
726
+ set((s) => ({
727
+ rows: s.rows.map((r) =>
728
+ r.id === row.id
729
+ ? {
730
+ ...r,
731
+ status: "error",
732
+ error: data.error || "Video generation failed",
733
+ }
734
+ : r,
735
+ ),
736
+ }));
737
+ break;
738
+ }
739
+ });
740
+
741
+ const vidRow = get().rows.find((r) => r.id === row.id);
742
+ if (!vidRow || vidRow.status === "error") {
743
+ continue;
744
+ }
745
+
746
+ // No script → no voiceover: the motion video is the final result.
747
+ if (!hasVoice) {
748
+ set((s) => ({
749
+ rows: s.rows.map((r) =>
750
+ r.id === row.id
751
+ ? { ...r, status: "done", result: vidRow.motionResult }
752
+ : r,
753
+ ),
754
+ }));
755
+ continue;
756
+ }
757
+
758
+ // ===== STEP 3: Mux audio + video =====
759
+ set((s) => ({
760
+ rows: s.rows.map((r) =>
761
+ r.id === row.id ? { ...r, status: "muxing" } : r,
762
+ ),
763
+ }));
764
+
765
+ const muxRes = await fetch(`${API_BASE}/api/render/mux-audio`, {
766
+ method: "POST",
767
+ headers: { "Content-Type": "application/json" },
768
+ body: JSON.stringify({
769
+ videoPath: normalizeImagePath(vidRow.motionResult!),
770
+ audioPath: normalizeImagePath(vidRow.audioResult!),
771
+ projectId,
772
+ }),
773
+ signal,
774
+ });
775
+
776
+ if (!muxRes.ok) {
777
+ const err = await muxRes.text();
778
+ set((s) => ({
779
+ rows: s.rows.map((r) =>
780
+ r.id === row.id ? { ...r, status: "error", error: err } : r,
781
+ ),
782
+ }));
783
+ continue;
784
+ }
785
+
786
+ let muxDone = false;
787
+ await readSSEStream(muxRes, (event, data) => {
788
+ if (muxDone) return;
789
+ switch (event) {
790
+ case "log":
791
+ set((s) => ({
792
+ rows: s.rows.map((r) =>
793
+ r.id === row.id
794
+ ? { ...r, logs: [...r.logs, data.text as string] }
795
+ : r,
796
+ ),
797
+ logs: [
798
+ ...s.logs,
799
+ `[${i + 1}/${targets.length}] ${data.text as string}`,
800
+ ],
801
+ }));
802
+ break;
803
+ case "complete":
804
+ muxDone = true;
805
+ set((s) => ({
806
+ rows: s.rows.map((r) =>
807
+ r.id === row.id
808
+ ? {
809
+ ...r,
810
+ status: "done",
811
+ result: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
812
+ }
813
+ : r,
814
+ ),
815
+ }));
816
+ break;
817
+ case "error":
818
+ muxDone = true;
819
+ set((s) => ({
820
+ rows: s.rows.map((r) =>
821
+ r.id === row.id
822
+ ? {
823
+ ...r,
824
+ status: "error",
825
+ error: data.error || "Muxing failed",
826
+ }
827
+ : r,
828
+ ),
829
+ }));
830
+ break;
831
+ }
832
+ });
833
+ } catch (e: any) {
834
+ if (e?.name === "AbortError") {
835
+ cancelled = true;
836
+ break;
837
+ }
838
+ set((s) => ({
839
+ rows: s.rows.map((r) =>
840
+ r.id === row.id
841
+ ? { ...r, status: "error", error: String(e) }
842
+ : r,
843
+ ),
844
+ }));
845
+ }
846
+ }
847
+
848
+ batchAbortController = null;
849
+ set({ running: false, progress: null, cancelRequested: false });
850
+
851
+ if (!cancelled) {
852
+ playBeep();
853
+ }
854
+ },
855
+
856
+ generateRow: (projectId, id) => get().generateRows(projectId, [id]),
857
+
858
+ generateAll: (projectId) => {
859
+ const ids = get()
860
+ .rows.filter(
861
+ (r) =>
862
+ r.prompt.trim() &&
863
+ r.imagePath &&
864
+ (!r.script.trim() || get().voiceRefPath),
865
+ )
866
+ .map((r) => r.id);
867
+ return get().generateRows(projectId, ids);
868
+ },
869
+
870
+ cancel: () => {
871
+ set({ cancelRequested: true });
872
+ if (batchAbortController) {
873
+ batchAbortController.abort();
874
+ }
875
+ set({
876
+ running: false,
877
+ progress: null,
878
+ rows: get().rows.map((r) =>
879
+ r.status === "tts" || r.status === "video" || r.status === "muxing"
880
+ ? { ...r, status: "idle" }
881
+ : r,
882
+ ),
883
+ });
884
+ // Keep cancelRequested: true — the loop's top-of-iteration check breaks out
885
+ // between rows, and generateRows resets the flag in its final set().
886
+ fetch(`${API_BASE}/api/render/cancel`, { method: "POST" }).catch(() => {});
887
+ },
888
+
889
+ stitchVideos: async () => {
890
+ if (get().stitching) return;
891
+
892
+ const results = get()
893
+ .rows.filter((r) => r.result)
894
+ .map((r) => r.result as string);
895
+
896
+ if (results.length < 2) {
897
+ set({
898
+ stitchError: "Need at least 2 generated videos to stitch.",
899
+ stitchResult: null,
900
+ });
901
+ return;
902
+ }
903
+
904
+ set({
905
+ stitching: true,
906
+ stitchLogs: [],
907
+ stitchResult: null,
908
+ stitchError: null,
909
+ });
910
+
911
+ try {
912
+ const ffmpeg = await loadFFmpeg();
913
+
914
+ // Write each generated video into the in-memory FS in order.
915
+ const list: string[] = [];
916
+ for (let i = 0; i < results.length; i++) {
917
+ const blob = await fetch(results[i]).then((r) => {
918
+ if (!r.ok) throw new Error(`Failed to fetch video ${i + 1}`);
919
+ return r.blob();
920
+ });
921
+ const data = new Uint8Array(await blob.arrayBuffer());
922
+ const name = `in${i}.mp4`;
923
+ await ffmpeg.writeFile(name, data);
924
+ list.push(`file '${name}'`);
925
+ set((s) => ({
926
+ stitchLogs: [...s.stitchLogs, `Added video ${i + 1}/${results.length}`],
927
+ }));
928
+ }
929
+
930
+ await ffmpeg.writeFile("list.txt", list.join("\n"));
931
+ set((s) => ({
932
+ stitchLogs: [...s.stitchLogs, "Concatenating videos..."],
933
+ }));
934
+
935
+ const ret = await ffmpeg.exec([
936
+ "-f",
937
+ "concat",
938
+ "-safe",
939
+ "0",
940
+ "-i",
941
+ "list.txt",
942
+ "-c",
943
+ "copy",
944
+ "output.mp4",
945
+ ]);
946
+
947
+ if (ret !== 0) {
948
+ throw new Error(`ffmpeg exited with code ${ret}`);
949
+ }
950
+
951
+ const out = (await ffmpeg.readFile("output.mp4")) as Uint8Array;
952
+ const bytes = new Uint8Array(out);
953
+ const url = URL.createObjectURL(
954
+ new Blob([bytes.buffer], { type: "video/mp4" }),
955
+ );
956
+
957
+ set({
958
+ stitching: false,
959
+ stitchResult: url,
960
+ stitchLogs: [...get().stitchLogs, "Done"],
961
+ });
962
+ } catch (e) {
963
+ set({
964
+ stitching: false,
965
+ stitchError: String(e),
966
+ });
967
+ }
968
+ },
969
+
970
+ reset: () =>
971
+ set({
972
+ rows: [makeRow()],
973
+ duration: 5,
974
+ aspectRatio: "1:1",
975
+ resolution: "480p",
976
+ mode: "distilled",
977
+ quality: "high",
978
+ voiceRefPath: null,
979
+ voiceRefUrl: null,
980
+ voiceRefFilename: null,
981
+ running: false,
982
+ progress: null,
983
+ cancelRequested: false,
984
+ logs: [],
985
+ stitching: false,
986
+ stitchLogs: [],
987
+ stitchResult: null,
988
+ stitchError: null,
989
+ }),
990
+ }));