@effectnode/media 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/bin/effectnode-media +2 -0
  2. package/dist/backend/index.d.ts +8 -0
  3. package/dist/backend/index.js +13 -0
  4. package/dist/backend/movie-backend/agent/agent-backend.d.ts +16 -0
  5. package/dist/backend/movie-backend/agent/agent-backend.js +703 -0
  6. package/dist/backend/movie-backend/agent/storyprompt.d.ts +1 -0
  7. package/dist/backend/movie-backend/agent/storyprompt.js +90 -0
  8. package/dist/backend/movie-backend/agent/tools/edit-image.d.ts +3 -0
  9. package/dist/backend/movie-backend/agent/tools/edit-image.js +192 -0
  10. package/dist/backend/movie-backend/agent/tools/get-time.d.ts +3 -0
  11. package/dist/backend/movie-backend/agent/tools/get-time.js +7 -0
  12. package/dist/backend/movie-backend/agent/tools/grep-files.d.ts +3 -0
  13. package/dist/backend/movie-backend/agent/tools/grep-files.js +59 -0
  14. package/dist/backend/movie-backend/agent/tools/image-to-video-generation.d.ts +3 -0
  15. package/dist/backend/movie-backend/agent/tools/image-to-video-generation.js +160 -0
  16. package/dist/backend/movie-backend/agent/tools/index.d.ts +7 -0
  17. package/dist/backend/movie-backend/agent/tools/index.js +59 -0
  18. package/dist/backend/movie-backend/agent/tools/list-files.d.ts +3 -0
  19. package/dist/backend/movie-backend/agent/tools/list-files.js +17 -0
  20. package/dist/backend/movie-backend/agent/tools/read-file.d.ts +3 -0
  21. package/dist/backend/movie-backend/agent/tools/read-file.js +28 -0
  22. package/dist/backend/movie-backend/agent/tools/remove-file.d.ts +3 -0
  23. package/dist/backend/movie-backend/agent/tools/remove-file.js +22 -0
  24. package/dist/backend/movie-backend/agent/tools/rename-file.d.ts +3 -0
  25. package/dist/backend/movie-backend/agent/tools/rename-file.js +37 -0
  26. package/dist/backend/movie-backend/agent/tools/show-image.d.ts +3 -0
  27. package/dist/backend/movie-backend/agent/tools/show-image.js +28 -0
  28. package/dist/backend/movie-backend/agent/tools/stitch-videos.d.ts +3 -0
  29. package/dist/backend/movie-backend/agent/tools/stitch-videos.js +96 -0
  30. package/dist/backend/movie-backend/agent/tools/text-to-video-generation.d.ts +3 -0
  31. package/dist/backend/movie-backend/agent/tools/text-to-video-generation.js +125 -0
  32. package/dist/backend/movie-backend/agent/tools/types.d.ts +15 -0
  33. package/dist/backend/movie-backend/agent/tools/types.js +1 -0
  34. package/dist/backend/movie-backend/agent/tools/update-file.d.ts +3 -0
  35. package/dist/backend/movie-backend/agent/tools/update-file.js +27 -0
  36. package/dist/backend/movie-backend/agent/tools/write-file.d.ts +3 -0
  37. package/dist/backend/movie-backend/agent/tools/write-file.js +26 -0
  38. package/dist/backend/movie-backend/agent/workspace.d.ts +25 -0
  39. package/dist/backend/movie-backend/agent/workspace.js +124 -0
  40. package/dist/backend/movie-backend/core.d.ts +4 -0
  41. package/dist/backend/movie-backend/core.js +1167 -0
  42. package/dist/backend/movie-backend/generation-queue.d.ts +24 -0
  43. package/dist/backend/movie-backend/generation-queue.js +766 -0
  44. package/dist/backend/movie-backend/process.d.ts +21 -0
  45. package/dist/backend/movie-backend/process.js +98 -0
  46. package/dist/backend/movie-backend/render-media.d.ts +48 -0
  47. package/dist/backend/movie-backend/render-media.js +3311 -0
  48. package/dist/backend/routes/health.d.ts +1 -0
  49. package/dist/backend/routes/health.js +9 -0
  50. package/dist/backend/routes/index.d.ts +1 -0
  51. package/dist/backend/routes/index.js +11 -0
  52. package/dist/backend/routes/media.d.ts +1 -0
  53. package/dist/backend/routes/media.js +13 -0
  54. package/dist/backend/routes/projects.d.ts +8 -0
  55. package/dist/backend/routes/projects.js +68 -0
  56. package/dist/backend/routes/voice.d.ts +1 -0
  57. package/dist/backend/routes/voice.js +39 -0
  58. package/dist/backend/routes.d.ts +1 -0
  59. package/dist/backend/routes.js +20 -0
  60. package/dist/backend/server.d.ts +1 -0
  61. package/dist/backend/server.js +23 -0
  62. package/dist/backend/workspace.d.ts +5 -0
  63. package/dist/backend/workspace.js +27 -0
  64. package/dist/backend/ws/index.d.ts +3 -0
  65. package/dist/backend/ws/index.js +27 -0
  66. package/dist/backend/ws/messages.d.ts +8 -0
  67. package/dist/backend/ws/messages.js +21 -0
  68. package/dist/backend/ws.d.ts +3 -0
  69. package/dist/backend/ws.js +29 -0
  70. package/dist/index.d.ts +2 -0
  71. package/dist/index.js +80 -0
  72. package/frontend/index.html +22 -0
  73. package/frontend/public/ffmpeg/ffmpeg-core.js +16 -0
  74. package/frontend/public/ffmpeg/ffmpeg-core.wasm +0 -0
  75. package/frontend/src/index.css +46 -0
  76. package/frontend/src/main.tsx +13 -0
  77. package/frontend/src/movie-app/AppRouter.tsx +14 -0
  78. package/frontend/src/movie-app/MediaStudio.tsx +9 -0
  79. package/frontend/src/movie-app/SetupPage.tsx +293 -0
  80. package/frontend/src/movie-app/components/AgentWorkspace.tsx +574 -0
  81. package/frontend/src/movie-app/components/ChatUI/ChatUI.tsx +655 -0
  82. package/frontend/src/movie-app/components/ChatUI/MarkdownMessage.tsx +267 -0
  83. package/frontend/src/movie-app/components/EditorTabs/AgentTab.tsx +61 -0
  84. package/frontend/src/movie-app/components/EditorTabs/BatchImageToVideoTab.tsx +927 -0
  85. package/frontend/src/movie-app/components/EditorTabs/BatchVideoTab.tsx +777 -0
  86. package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +913 -0
  87. package/frontend/src/movie-app/components/EditorTabs/CharacterSheet.tsx +234 -0
  88. package/frontend/src/movie-app/components/EditorTabs/CharactersTab.tsx +664 -0
  89. package/frontend/src/movie-app/components/EditorTabs/CropTool.tsx +251 -0
  90. package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +305 -0
  91. package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +249 -0
  92. package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +555 -0
  93. package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +722 -0
  94. package/frontend/src/movie-app/components/EditorTabs/LlmServerTab.tsx +40 -0
  95. package/frontend/src/movie-app/components/EditorTabs/MlxVlmServerPanel.tsx +490 -0
  96. package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +938 -0
  97. package/frontend/src/movie-app/components/EditorTabs/ReferencesToVideoTab.tsx +881 -0
  98. package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +267 -0
  99. package/frontend/src/movie-app/components/EditorTabs/StoryWriterTab.tsx +51 -0
  100. package/frontend/src/movie-app/components/EditorTabs/TaskQueuePanel.tsx +246 -0
  101. package/frontend/src/movie-app/components/EditorTabs/TerminalLogPanel.tsx +49 -0
  102. package/frontend/src/movie-app/components/EditorTabs/TextToImageTab.tsx +574 -0
  103. package/frontend/src/movie-app/components/EditorTabs/VoiceChatPanel.tsx +320 -0
  104. package/frontend/src/movie-app/components/ProjectEditorPage.tsx +636 -0
  105. package/frontend/src/movie-app/components/ProjectManager.tsx +381 -0
  106. package/frontend/src/movie-app/index.css +46 -0
  107. package/frontend/src/movie-app/index.html +16 -0
  108. package/frontend/src/movie-app/lambobo.png +0 -0
  109. package/frontend/src/movie-app/lib/batchImageToVideoStorage.ts +69 -0
  110. package/frontend/src/movie-app/lib/batchVideoStorage.ts +68 -0
  111. package/frontend/src/movie-app/lib/batchVoiceStorage.ts +75 -0
  112. package/frontend/src/movie-app/lib/ffmpeg.ts +42 -0
  113. package/frontend/src/movie-app/lib/referencesToVideoStorage.ts +64 -0
  114. package/frontend/src/movie-app/lib/textToImageStorage.ts +61 -0
  115. package/frontend/src/movie-app/public/ffmpeg/ffmpeg-core.js +16 -0
  116. package/frontend/src/movie-app/public/ffmpeg/ffmpeg-core.wasm +0 -0
  117. package/frontend/src/movie-app/stores/batchImageToVideoStore.ts +736 -0
  118. package/frontend/src/movie-app/stores/batchVideoStore.ts +670 -0
  119. package/frontend/src/movie-app/stores/batchVoiceStore.ts +990 -0
  120. package/frontend/src/movie-app/stores/characterStore.ts +109 -0
  121. package/frontend/src/movie-app/stores/chatStore.ts +459 -0
  122. package/frontend/src/movie-app/stores/generationStore.ts +2140 -0
  123. package/frontend/src/movie-app/stores/logStore.ts +25 -0
  124. package/frontend/src/movie-app/stores/movieStudioStore.ts +709 -0
  125. package/frontend/src/movie-app/stores/projectStore.ts +124 -0
  126. package/frontend/src/movie-app/stores/queueStore.ts +214 -0
  127. package/frontend/src/movie-app/stores/referencesToVideoStore.ts +467 -0
  128. package/frontend/src/movie-app/stores/sceneVisualStore.ts +251 -0
  129. package/frontend/src/movie-app/stores/voiceChatStore.ts +170 -0
  130. package/frontend/src/movie-app/stores/workspaceStore.ts +158 -0
  131. package/frontend/src/vite-env.d.ts +1 -0
  132. package/frontend/tsconfig.json +19 -0
  133. package/package.json +65 -0
@@ -0,0 +1,3311 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync, readdirSync, statSync, unlinkSync, rmSync, copyFileSync, } from "node:fs";
2
+ import { randomUUID } from "node:crypto";
3
+ import { homedir } from "node:os";
4
+ import { join, sep } from "node:path";
5
+ import { spawn, whichSync, mimeType } from "./process";
6
+ // Track the currently active spawn process so it can be cancelled
7
+ let activeProc = null;
8
+ // Track the long-running mlx-vlm server process (separate from render jobs).
9
+ let agentServerProc = null;
10
+ let agentStopRequested = false;
11
+ let agentServerPort = null;
12
+ /** Port the app started the mlx-vlm server on (null when the server is not running). */
13
+ export function getAgentServerPort() {
14
+ return agentServerPort;
15
+ }
16
+ const APP_DATA_DIR = join(homedir(), "media-studio");
17
+ const OUTPUT_DIR = join(APP_DATA_DIR, "output");
18
+ const UPLOAD_DIR = join(APP_DATA_DIR, "upload");
19
+ const AGENT_UPLOAD_DIR = join(APP_DATA_DIR, "agent-upload");
20
+ const EXTRACTED_FRAMES_DIR = join(APP_DATA_DIR, "extracted-frames");
21
+ const CHARACTER_SHEET_DIR = join(APP_DATA_DIR, "character-sheet");
22
+ const AGENTS_DIR = join(APP_DATA_DIR, "agents");
23
+ const JSON_DIR = join(APP_DATA_DIR, "json");
24
+ const PYTHON_DIR = join(APP_DATA_DIR, "python-src");
25
+ const TEMP_DIR = join(APP_DATA_DIR, "temp");
26
+ const PROJECTS_FILE = join(JSON_DIR, "projects.json");
27
+ const CHARACTERS_FILE = join(JSON_DIR, "characters.json");
28
+ const MLXGEN_MODEL = "AbstractFramework/qwen-image-edit-2511-8bit";
29
+ const Z_IMAGE_MODEL = "AbstractFramework/z-image-turbo-8bit";
30
+ const FLUX_KLEIN_MODEL = "AbstractFramework/flux.2-klein-4b-8bit";
31
+ const MLX_VLM_MODEL = "mlx-community/gemma-4-e2b-it-4bit";
32
+ const H3_MODEL = "appautomaton/minimax-h3-base-8bit-mlx";
33
+ const VIDEO_STAGE_FLAGS = {
34
+ distilled: "--distilled",
35
+ "one-stage": "--one-stage",
36
+ "two-stage": "--two-stage",
37
+ };
38
+ const TTS_MODELS = {
39
+ low: "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
40
+ high: "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
41
+ };
42
+ /** Resolve the CLI stage flag for a video generation mode (defaults to distilled). */
43
+ function stageFlagFor(mode) {
44
+ return typeof mode === "string"
45
+ ? (VIDEO_STAGE_FLAGS[mode] ?? "--distilled")
46
+ : "--distilled";
47
+ }
48
+ // ========== Project Helpers ==========
49
+ function ensureDir(dir) {
50
+ if (!existsSync(dir)) {
51
+ mkdirSync(dir, { recursive: true });
52
+ }
53
+ }
54
+ const PROJECT_ID_RE = /^[a-zA-Z0-9_-]{1,64}$/;
55
+ function isValidProjectId(id) {
56
+ return PROJECT_ID_RE.test(id);
57
+ }
58
+ /**
59
+ * Resolve an output directory for generated media. Returns the directory path,
60
+ * or null when the project id or supplied directory is invalid / outside the
61
+ * allowed roots.
62
+ */
63
+ function resolveOutputDir(outputDir, projectId) {
64
+ if (!isValidProjectId(projectId))
65
+ return null;
66
+ const base = typeof outputDir === "string" && outputDir.trim()
67
+ ? outputDir.trim()
68
+ : join(OUTPUT_DIR, projectId);
69
+ // Defense in depth: reject null bytes and `..` traversal segments.
70
+ if (base.includes("\0") || base.split(/[/\\]/).includes(".."))
71
+ return null;
72
+ ensureDir(base);
73
+ let realBase;
74
+ try {
75
+ realBase = realpathSync(base);
76
+ }
77
+ catch {
78
+ return null;
79
+ }
80
+ for (const root of [OUTPUT_DIR, UPLOAD_DIR, AGENTS_DIR]) {
81
+ ensureDir(root);
82
+ const realRoot = realpathSync(root);
83
+ if (realBase === realRoot || realBase.startsWith(realRoot + sep)) {
84
+ return base;
85
+ }
86
+ }
87
+ return null;
88
+ }
89
+ function readProjects() {
90
+ ensureDir(JSON_DIR);
91
+ if (!existsSync(PROJECTS_FILE)) {
92
+ writeFileSync(PROJECTS_FILE, "[]", "utf-8");
93
+ return [];
94
+ }
95
+ try {
96
+ const raw = readFileSync(PROJECTS_FILE, "utf-8");
97
+ return JSON.parse(raw);
98
+ }
99
+ catch {
100
+ return [];
101
+ }
102
+ }
103
+ function writeProjects(projects) {
104
+ ensureDir(JSON_DIR);
105
+ writeFileSync(PROJECTS_FILE, JSON.stringify(projects, null, 2), "utf-8");
106
+ }
107
+ function readCharacters() {
108
+ ensureDir(JSON_DIR);
109
+ if (!existsSync(CHARACTERS_FILE)) {
110
+ writeFileSync(CHARACTERS_FILE, "[]", "utf-8");
111
+ return [];
112
+ }
113
+ try {
114
+ return JSON.parse(readFileSync(CHARACTERS_FILE, "utf-8"));
115
+ }
116
+ catch {
117
+ return [];
118
+ }
119
+ }
120
+ function writeCharacters(characters) {
121
+ ensureDir(JSON_DIR);
122
+ writeFileSync(CHARACTERS_FILE, JSON.stringify(characters, null, 2), "utf-8");
123
+ }
124
+ function makeId() {
125
+ return Date.now().toString(36) + "-" + Math.random().toString(36).slice(2, 8);
126
+ }
127
+ function openInFinder(dirPath) {
128
+ // Use Bun.spawn instead of execSync — non-blocking and native to Bun
129
+ spawn(["open", dirPath], {
130
+ stdout: "ignore",
131
+ stderr: "ignore",
132
+ onExit: (_proc, exitCode, _signalCode, _error) => {
133
+ if (exitCode !== 0) {
134
+ console.error(`openInFinder: "open ${dirPath}" exited with code ${exitCode}`);
135
+ }
136
+ },
137
+ });
138
+ }
139
+ // ========== SSE Helper ==========
140
+ // Allowed directories for file serving and path resolution.
141
+ // Lazily resolved on first use because dirs may not exist at import time.
142
+ let _allowedRealDirs = null;
143
+ function getAllowedRealDirs() {
144
+ if (_allowedRealDirs)
145
+ return _allowedRealDirs;
146
+ [OUTPUT_DIR, UPLOAD_DIR, AGENT_UPLOAD_DIR, CHARACTER_SHEET_DIR].forEach((d) => ensureDir(d));
147
+ _allowedRealDirs = [
148
+ realpathSync(OUTPUT_DIR) + sep,
149
+ realpathSync(UPLOAD_DIR) + sep,
150
+ realpathSync(AGENT_UPLOAD_DIR) + sep,
151
+ realpathSync(CHARACTER_SHEET_DIR) + sep,
152
+ ];
153
+ return _allowedRealDirs;
154
+ }
155
+ /** Validate that `resolvedPath` is inside an allowed directory. */
156
+ function isPathAllowed(resolvedPath) {
157
+ return getAllowedRealDirs().some((dir) => resolvedPath.startsWith(dir));
158
+ }
159
+ /** Resolve and validate a user-supplied filename. Looks in both upload and output dirs. */
160
+ function resolveSafePath(candidate, projectId) {
161
+ // Reject anything that looks like a path — only bare filenames allowed
162
+ const base = candidate.split(/[/\\]/).pop() || candidate;
163
+ if (base !== candidate || base.includes("..") || base.startsWith(".")) {
164
+ return null;
165
+ }
166
+ // Try the agent-upload dir first, then upload and output dirs.
167
+ for (const dir of [AGENT_UPLOAD_DIR, UPLOAD_DIR, OUTPUT_DIR]) {
168
+ const candidatePath = join(dir, projectId, base);
169
+ if (existsSync(candidatePath)) {
170
+ const resolved = realpathSync(candidatePath);
171
+ if (isPathAllowed(resolved))
172
+ return resolved;
173
+ }
174
+ }
175
+ // TTS voiceovers are stored under <output>/<projectId>/voices/<id>/. Search
176
+ // each voice folder for the basename so muxing can resolve the audio.
177
+ const voicesRoot = join(OUTPUT_DIR, projectId, "voices");
178
+ let voiceSubdirs = [];
179
+ try {
180
+ voiceSubdirs = readdirSync(voicesRoot).map((entry) => join(voicesRoot, entry));
181
+ }
182
+ catch {
183
+ // voices dir does not exist yet — nothing to search
184
+ }
185
+ for (const voiceDir of voiceSubdirs) {
186
+ const candidatePath = join(voiceDir, base);
187
+ if (existsSync(candidatePath)) {
188
+ const resolved = realpathSync(candidatePath);
189
+ if (isPathAllowed(resolved))
190
+ return resolved;
191
+ }
192
+ }
193
+ return null;
194
+ }
195
+ /** Resolve and validate a user-supplied video filename. Only bare .mp4 names in the output dir. */
196
+ function resolveSafeVideoPath(candidate, projectId) {
197
+ const base = candidate.split(/[/\\]/).pop() || candidate;
198
+ if (base !== candidate || base.includes("..") || base.startsWith(".")) {
199
+ return null;
200
+ }
201
+ if (!base.toLowerCase().endsWith(".mp4"))
202
+ return null;
203
+ const candidatePath = join(OUTPUT_DIR, projectId, base);
204
+ if (!existsSync(candidatePath))
205
+ return null;
206
+ const resolved = realpathSync(candidatePath);
207
+ if (isPathAllowed(resolved))
208
+ return resolved;
209
+ return null;
210
+ }
211
+ /**
212
+ * Resolve the `mlxgen` executable installed via `uv tool install --upgrade mlx-gen`.
213
+ * uv tool installs binaries into `~/.local/bin`; fall back to relying on PATH.
214
+ */
215
+ async function getMlxgenBin() {
216
+ const candidates = [
217
+ join(homedir(), ".local", "bin", "mlxgen"),
218
+ "/opt/homebrew/bin/mlxgen",
219
+ "/usr/local/bin/mlxgen",
220
+ ];
221
+ for (const p of candidates) {
222
+ if (existsSync(p))
223
+ return p;
224
+ }
225
+ return "mlxgen";
226
+ }
227
+ /** True when the `mlxgen` executable is installed (known paths or PATH). */
228
+ function isMlxgenInstalled() {
229
+ const candidates = [
230
+ join(homedir(), ".local", "bin", "mlxgen"),
231
+ "/opt/homebrew/bin/mlxgen",
232
+ "/usr/local/bin/mlxgen",
233
+ ];
234
+ for (const p of candidates) {
235
+ if (existsSync(p))
236
+ return true;
237
+ }
238
+ // Fall back to PATH lookup (Bun native, synchronous).
239
+ try {
240
+ return whichSync("mlxgen") !== null;
241
+ }
242
+ catch {
243
+ return false;
244
+ }
245
+ }
246
+ /**
247
+ * Resolve the `mlx_vlm.server` executable installed via `uv tool install mlx-vlm`.
248
+ * uv tool installs binaries into `~/.local/bin`; fall back to relying on PATH.
249
+ */
250
+ async function getMlxVlmServerBin() {
251
+ const candidates = [
252
+ join(homedir(), ".local", "bin", "mlx_vlm.server"),
253
+ "/opt/homebrew/bin/mlx_vlm.server",
254
+ "/usr/local/bin/mlx_vlm.server",
255
+ ];
256
+ for (const p of candidates) {
257
+ if (existsSync(p))
258
+ return p;
259
+ }
260
+ return "mlx_vlm.server";
261
+ }
262
+ /** Kill any process listening on the given port (returns the PIDs that were killed). */
263
+ async function killProcessOnPort(port) {
264
+ try {
265
+ const proc = spawn(["lsof", "-ti", `:${port}`], {
266
+ stdout: "pipe",
267
+ stderr: "pipe",
268
+ });
269
+ const output = await new Response(proc.stdout).text();
270
+ const exitCode = await proc.exited;
271
+ if (exitCode !== 0)
272
+ return [];
273
+ const pids = output.trim().split(/\s+/).filter(Boolean);
274
+ for (const pid of pids) {
275
+ spawn(["kill", "-9", pid], { stdout: "ignore", stderr: "ignore" });
276
+ }
277
+ return pids;
278
+ }
279
+ catch {
280
+ return [];
281
+ }
282
+ }
283
+ /** True when the `mlx_vlm.server` executable is installed (known paths or PATH). */
284
+ function isMlxVlmInstalled() {
285
+ const candidates = [
286
+ join(homedir(), ".local", "bin", "mlx_vlm.server"),
287
+ "/opt/homebrew/bin/mlx_vlm.server",
288
+ "/usr/local/bin/mlx_vlm.server",
289
+ ];
290
+ for (const p of candidates) {
291
+ if (existsSync(p))
292
+ return true;
293
+ }
294
+ try {
295
+ return whichSync("mlx_vlm.server") !== null;
296
+ }
297
+ catch {
298
+ return false;
299
+ }
300
+ }
301
+ /**
302
+ * Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel).
303
+ * Falls back to relying on PATH when neither known location exists.
304
+ */
305
+ async function getFfmpegBin() {
306
+ const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
307
+ for (const p of candidates) {
308
+ if (existsSync(p))
309
+ return p;
310
+ }
311
+ return "ffmpeg";
312
+ }
313
+ /** Fixed filename mlx_audio.tts.generate writes its output clip as. */
314
+ const TTS_OUTPUT_FILENAME = "audio_000.mp3";
315
+ /**
316
+ * Resolve the TTS output clip for a voice folder. mlx_audio.tts.generate always
317
+ * writes its single generated clip as `audio_000.mp3` directly into `--output`,
318
+ * so the path is deterministic and there is no need to scan for a newest file.
319
+ */
320
+ function resolveAudioFile(root) {
321
+ const path = join(root, TTS_OUTPUT_FILENAME);
322
+ return existsSync(path) ? path : null;
323
+ }
324
+ /** Directory where Hugging Face Hub caches downloaded models. */
325
+ function huggingfaceCacheDir() {
326
+ if (process.env.HF_HUB_CACHE)
327
+ return process.env.HF_HUB_CACHE;
328
+ if (process.env.HF_HOME)
329
+ return join(process.env.HF_HOME, "hub");
330
+ return join(homedir(), ".cache", "huggingface", "hub");
331
+ }
332
+ /** True when the given MLX-Gen model has already been downloaded to the HF cache. */
333
+ function isModelDownloaded(model = MLXGEN_MODEL) {
334
+ const modelDirName = `models--${model.replace("/", "--")}`;
335
+ const snapshotsDir = join(huggingfaceCacheDir(), modelDirName, "snapshots");
336
+ if (!existsSync(snapshotsDir))
337
+ return false;
338
+ try {
339
+ return readdirSync(snapshotsDir).some((name) => {
340
+ try {
341
+ return statSync(join(snapshotsDir, name)).isDirectory();
342
+ }
343
+ catch {
344
+ return false;
345
+ }
346
+ });
347
+ }
348
+ catch {
349
+ return false;
350
+ }
351
+ }
352
+ /** Directory the H3 model weights are downloaded into (mlx-h3/weights). */
353
+ function h3WeightsDir() {
354
+ return join(PYTHON_DIR, "mlx-h3", "weights");
355
+ }
356
+ /** True when the H3 model has been downloaded into <mlx-h3>/weights. */
357
+ function isH3ModelDownloaded() {
358
+ const dir = h3WeightsDir();
359
+ if (!existsSync(dir))
360
+ return false;
361
+ try {
362
+ return readdirSync(dir).length > 0;
363
+ }
364
+ catch {
365
+ return false;
366
+ }
367
+ }
368
+ /**
369
+ * Strip ANSI escape sequences (colors, cursor control, etc.) from a string.
370
+ * Tools like `uv` emit these when streaming to a TTY; forwarding them to the
371
+ * browser renders them as raw control glyphs (e.g. `␛[2m`) instead of text.
372
+ */
373
+ function stripAnsi(input) {
374
+ // Matches CSI (`ESC [ ...`) and other two-byte control sequences (`ESC @`–`ESC _`).
375
+ // eslint-disable-next-line no-control-regex
376
+ return input.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "");
377
+ }
378
+ /** Stream stdout/stderr from a Bun subprocess to an SSE response. */
379
+ async function streamToSSE(readable, prefix, send) {
380
+ let text = "";
381
+ const reader = readable?.getReader();
382
+ if (!reader)
383
+ return text;
384
+ const decoder = new TextDecoder();
385
+ try {
386
+ while (true) {
387
+ const { done, value } = await reader.read();
388
+ if (done)
389
+ break;
390
+ // Use { stream: true } so multi-byte UTF-8 characters split across
391
+ // chunks are reassembled correctly instead of producing mojibake.
392
+ const chunk = stripAnsi(decoder.decode(value, { stream: true }));
393
+ if (chunk) {
394
+ console.log(`[${prefix}]`, chunk);
395
+ text += chunk;
396
+ send("log", { text: chunk });
397
+ }
398
+ }
399
+ // Flush any remaining bytes buffered in the decoder.
400
+ const final = stripAnsi(decoder.decode());
401
+ if (final) {
402
+ console.log(`[${prefix}]`, final);
403
+ text += final;
404
+ send("log", { text: final });
405
+ }
406
+ }
407
+ finally {
408
+ reader.releaseLock();
409
+ }
410
+ return text;
411
+ }
412
+ /** Read a process stream, decoding UTF-8 and stripping ANSI, emitting chunks. */
413
+ async function readStream(readable, onChunk) {
414
+ const reader = readable?.getReader();
415
+ if (!reader)
416
+ return;
417
+ const decoder = new TextDecoder();
418
+ try {
419
+ while (true) {
420
+ const { done, value } = await reader.read();
421
+ if (done)
422
+ break;
423
+ const chunk = stripAnsi(decoder.decode(value, { stream: true }));
424
+ if (chunk)
425
+ onChunk(chunk);
426
+ }
427
+ const final = stripAnsi(decoder.decode());
428
+ if (final)
429
+ onChunk(final);
430
+ }
431
+ finally {
432
+ reader.releaseLock();
433
+ }
434
+ }
435
+ /** Run a command to completion, returning its exit status and combined output. */
436
+ async function runCommand(args, opts = {}) {
437
+ const proc = spawn(args, {
438
+ env: process.env,
439
+ stdout: "pipe",
440
+ stderr: "pipe",
441
+ ...(opts.cwd ? { cwd: opts.cwd } : {}),
442
+ });
443
+ activeProc = proc;
444
+ let output = "";
445
+ const onLog = opts.onLog;
446
+ const collect = (text) => {
447
+ output += text;
448
+ if (onLog)
449
+ onLog(text);
450
+ };
451
+ await Promise.all([
452
+ readStream(proc.stdout, collect),
453
+ readStream(proc.stderr, collect),
454
+ ]);
455
+ const exitCode = await proc.exited;
456
+ return { success: exitCode === 0, output: output.trim() };
457
+ }
458
+ /** Copy a generated file into the project's backup folder with a timestamped name. */
459
+ function backupFile(sourcePath, projectId) {
460
+ try {
461
+ const backupDir = join(OUTPUT_DIR, String(projectId), "backup");
462
+ ensureDir(backupDir);
463
+ const base = sourcePath.split(sep).pop() || "file";
464
+ const dot = base.lastIndexOf(".");
465
+ const stem = dot > 0 ? base.slice(0, dot) : base;
466
+ const ext = dot > 0 ? base.slice(dot) : "";
467
+ const ts = new Date().toISOString().replace(/[:.]/g, "-");
468
+ const backupPath = join(backupDir, `${stem}-${ts}${ext}`);
469
+ copyFileSync(sourcePath, backupPath);
470
+ return backupPath;
471
+ }
472
+ catch {
473
+ return null;
474
+ }
475
+ }
476
+ /** Kill the currently active spawn process (used to cancel a running task). */
477
+ export function cancelActiveRender() {
478
+ if (activeProc) {
479
+ try {
480
+ activeProc.kill();
481
+ }
482
+ catch {
483
+ // process may already be dead
484
+ }
485
+ activeProc = null;
486
+ }
487
+ }
488
+ /** Generate a single character/place image (text-to-image) and back it up. */
489
+ export async function generateAssetImage(projectId, kind, slug, prompt, onLog) {
490
+ const mlxgen = await getMlxgenBin();
491
+ const outputDir = join(OUTPUT_DIR, projectId);
492
+ ensureDir(outputDir);
493
+ const outputFile = `${kind}-${slug}.png`;
494
+ const outputPath = join(outputDir, outputFile);
495
+ const result = await runCommand([
496
+ mlxgen,
497
+ "generate",
498
+ "--model",
499
+ Z_IMAGE_MODEL,
500
+ "--prompt",
501
+ prompt,
502
+ "--output",
503
+ outputPath,
504
+ "--steps",
505
+ "6",
506
+ "--width",
507
+ "1024",
508
+ "--height",
509
+ "1024",
510
+ ], { onLog });
511
+ if (!result.success || !existsSync(outputPath)) {
512
+ return { error: result.output || `Failed to generate ${kind} ${slug}` };
513
+ }
514
+ backupFile(outputPath, projectId);
515
+ return {
516
+ filename: outputFile,
517
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
518
+ };
519
+ }
520
+ /** Build the LTX image-to-video prompt for a scene (dialogue + voiceover). */
521
+ function buildVideoPrompt(scene, characters) {
522
+ const nameOf = (slug) => characters.find((c) => String(c?.slug) === String(slug))?.name ||
523
+ String(slug || "");
524
+ const lines = Array.isArray(scene.scriptLines)
525
+ ? scene.scriptLines
526
+ .map((l) => `${nameOf(l?.characterSlug)} says: "${l?.line || ""}"`)
527
+ .join(" ")
528
+ : "";
529
+ const vo = scene.voiceOver ? ` Voiceover: "${scene.voiceOver}"` : "";
530
+ return `${scene.description || ""} ${lines}${vo}`.trim();
531
+ }
532
+ /** Generate a single scene video (LTX-2.3) from its scene image, and back it up. */
533
+ export async function generateSceneVideo(uvPath, projectId, scene, characters, onLog) {
534
+ const s = String(scene?.slug || "")
535
+ .trim()
536
+ .replace(/[^a-zA-Z0-9_-]/g, "_");
537
+ if (!s)
538
+ return { error: "Invalid scene slug" };
539
+ const outputDir = join(OUTPUT_DIR, projectId);
540
+ const sceneImagePath = join(outputDir, `scene-${s}.png`);
541
+ if (!existsSync(sceneImagePath)) {
542
+ return {
543
+ error: `Scene image not found for "${s}". Generate scene images first.`,
544
+ };
545
+ }
546
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
547
+ const videoFile = `scene-${s}.mp4`;
548
+ const videoPath = join(outputDir, videoFile);
549
+ const frames = Math.max(1, Math.round((Number(scene?.duration) + 3) * 24));
550
+ const result = await runCommand([
551
+ uvPath,
552
+ "run",
553
+ "ltx-2-mlx",
554
+ "generate",
555
+ "--model",
556
+ "dgrauet/ltx-2.3-mlx-q8",
557
+ "--prompt",
558
+ buildVideoPrompt(scene, characters),
559
+ "--distilled",
560
+ "--frames",
561
+ String(frames),
562
+ "--width",
563
+ "320",
564
+ "--height",
565
+ "569",
566
+ "--frame-rate",
567
+ "24",
568
+ "--image",
569
+ sceneImagePath,
570
+ "--output",
571
+ videoPath,
572
+ ], { cwd: ltxFolder, onLog });
573
+ if (!result.success || !existsSync(videoPath)) {
574
+ return { error: result.output || `Failed to generate video ${s}` };
575
+ }
576
+ backupFile(videoPath, projectId);
577
+ return {
578
+ filename: videoFile,
579
+ url: `/api/files?path=${encodeURIComponent(videoPath)}`,
580
+ };
581
+ }
582
+ /** Normalize a value into a safe filesystem slug. */
583
+ function slugify(v) {
584
+ return String(v || "")
585
+ .trim()
586
+ .replace(/[^a-zA-Z0-9_-]/g, "_");
587
+ }
588
+ /**
589
+ * Generate a single scene image via fast-image-edit (FLUX.2 Klein), using the
590
+ * already-generated character/place images referenced by the scene's slugs.
591
+ */
592
+ export async function generateSceneImage(projectId, scene, onLog) {
593
+ const s = slugify(scene?.slug);
594
+ if (!s)
595
+ return { error: "Invalid scene slug" };
596
+ const outputDir = join(OUTPUT_DIR, projectId);
597
+ const mlxgen = await getMlxgenBin();
598
+ const refImages = [];
599
+ for (const cs of Array.isArray(scene?.characterSlugs)
600
+ ? scene.characterSlugs
601
+ : []) {
602
+ const f = join(outputDir, `character-${slugify(cs)}.png`);
603
+ if (existsSync(f))
604
+ refImages.push(f);
605
+ }
606
+ const placeFile = join(outputDir, `place-${slugify(scene?.placeSlug)}.png`);
607
+ if (existsSync(placeFile))
608
+ refImages.push(placeFile);
609
+ if (refImages.length === 0) {
610
+ return {
611
+ error: `Scene "${s}": no matching character/place images. Render assets first.`,
612
+ };
613
+ }
614
+ const sceneImageFile = `scene-${s}.png`;
615
+ const sceneImagePath = join(outputDir, sceneImageFile);
616
+ const fluxArgs = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
617
+ for (const p of refImages)
618
+ fluxArgs.push("--image", p);
619
+ fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", "5", "--seed", "42", "--width", "448", "--height", "796");
620
+ const result = await runCommand(fluxArgs, { onLog });
621
+ if (!result.success || !existsSync(sceneImagePath)) {
622
+ return { error: result.output || `Failed to generate scene image ${s}` };
623
+ }
624
+ backupFile(sceneImagePath, projectId);
625
+ return {
626
+ filename: sceneImageFile,
627
+ url: `/api/files?path=${encodeURIComponent(sceneImagePath)}`,
628
+ };
629
+ }
630
+ // ========== Routes ==========
631
+ export async function renderMediaRoutes({ app, getUvPath, }) {
632
+ // ========== Upload ==========
633
+ // Serve generated files so the browser can display them
634
+ app.get("/api/files", async (req, res) => {
635
+ const filePath = req.query.path;
636
+ if (!filePath) {
637
+ res.status(404).json({ error: "File not found" });
638
+ return;
639
+ }
640
+ // Resolve and verify the path is within allowed directories
641
+ let resolved;
642
+ try {
643
+ resolved = realpathSync(filePath);
644
+ }
645
+ catch {
646
+ res.status(404).json({ error: "File not found" });
647
+ return;
648
+ }
649
+ if (!isPathAllowed(resolved)) {
650
+ res.status(403).json({ error: "Access denied" });
651
+ return;
652
+ }
653
+ try {
654
+ const buffer = readFileSync(resolved);
655
+ res.setHeader("Content-Type", mimeType(resolved) || "application/octet-stream");
656
+ res.setHeader("Content-Length", String(buffer.length));
657
+ res.setHeader("Cache-Control", "no-cache");
658
+ res.end(buffer);
659
+ }
660
+ catch (e) {
661
+ res
662
+ .status(500)
663
+ .json({ error: "Failed to read file", details: String(e) });
664
+ }
665
+ });
666
+ app.post("/api/upload/image", async (req, res) => {
667
+ const { image, filename, projectId } = req.body || {};
668
+ if (!image) {
669
+ res.status(400).json({ error: "Image data is required (base64)" });
670
+ return;
671
+ }
672
+ if (!projectId) {
673
+ res.status(400).json({ error: "Project ID is required" });
674
+ return;
675
+ }
676
+ try {
677
+ // Decode base64 (strip data URL prefix if present)
678
+ const base64 = image.replace(/^data:image\/\w+;base64,/, "");
679
+ const buffer = Buffer.from(base64, "base64");
680
+ const projectUploadDir = join(UPLOAD_DIR, projectId);
681
+ ensureDir(projectUploadDir);
682
+ const safeName = (filename || `upload-${Date.now()}.png`).replace(/[^a-zA-Z0-9._-]/g, "_");
683
+ const filePath = join(projectUploadDir, safeName);
684
+ writeFileSync(filePath, buffer);
685
+ res.json({
686
+ success: true,
687
+ path: filePath,
688
+ filename: safeName,
689
+ size: buffer.length,
690
+ });
691
+ }
692
+ catch (e) {
693
+ res
694
+ .status(500)
695
+ .json({ error: "Failed to save image", details: String(e) });
696
+ }
697
+ });
698
+ app.post("/api/upload/video", async (req, res) => {
699
+ const { video, filename, projectId } = req.body || {};
700
+ if (!video) {
701
+ res.status(400).json({ error: "Video data is required (base64)" });
702
+ return;
703
+ }
704
+ if (!projectId || !isValidProjectId(String(projectId))) {
705
+ res.status(400).json({ error: "Invalid project ID" });
706
+ return;
707
+ }
708
+ try {
709
+ // Decode base64 (strip any data URL prefix if present)
710
+ const base64 = String(video).replace(/^data:[^;]+;base64,/, "");
711
+ const buffer = Buffer.from(base64, "base64");
712
+ const projectUploadDir = join(UPLOAD_DIR, String(projectId));
713
+ ensureDir(projectUploadDir);
714
+ const safeName = (filename || `upload-${Date.now()}.mp4`).replace(/[^a-zA-Z0-9._-]/g, "_");
715
+ const filePath = join(projectUploadDir, safeName);
716
+ writeFileSync(filePath, buffer);
717
+ res.json({
718
+ success: true,
719
+ path: filePath,
720
+ filename: safeName,
721
+ size: buffer.length,
722
+ });
723
+ }
724
+ catch (e) {
725
+ res
726
+ .status(500)
727
+ .json({ error: "Failed to save video", details: String(e) });
728
+ }
729
+ });
730
+ app.post("/api/upload/audio", async (req, res) => {
731
+ const { audio, filename, projectId } = req.body || {};
732
+ if (!audio) {
733
+ res.status(400).json({ error: "Audio data is required (base64)" });
734
+ return;
735
+ }
736
+ if (!projectId || !isValidProjectId(String(projectId))) {
737
+ res.status(400).json({ error: "Invalid project ID" });
738
+ return;
739
+ }
740
+ try {
741
+ // Decode base64 (strip data URL prefix if present)
742
+ const base64 = String(audio).replace(/^data:audio\/\w+;base64,/, "");
743
+ const buffer = Buffer.from(base64, "base64");
744
+ const projectUploadDir = join(UPLOAD_DIR, projectId);
745
+ ensureDir(projectUploadDir);
746
+ const safeName = (filename || `upload-${Date.now()}.mp3`).replace(/[^a-zA-Z0-9._-]/g, "_");
747
+ const filePath = join(projectUploadDir, safeName);
748
+ writeFileSync(filePath, buffer);
749
+ res.json({
750
+ success: true,
751
+ path: filePath,
752
+ filename: safeName,
753
+ size: buffer.length,
754
+ });
755
+ }
756
+ catch (e) {
757
+ res
758
+ .status(500)
759
+ .json({ error: "Failed to save audio", details: String(e) });
760
+ }
761
+ });
762
+ // List project images (from uploads and generated outputs)
763
+ app.get("/api/projects/:id/images", (req, res) => {
764
+ const { id } = req.params;
765
+ if (!isValidProjectId(id)) {
766
+ res.status(400).json({ error: "Invalid project ID" });
767
+ return;
768
+ }
769
+ const imageExts = new Set([
770
+ ".png",
771
+ ".jpg",
772
+ ".jpeg",
773
+ ".webp",
774
+ ".gif",
775
+ ".bmp",
776
+ ]);
777
+ const results = [];
778
+ for (const [source, dir] of [
779
+ ["upload", UPLOAD_DIR],
780
+ ["generated", OUTPUT_DIR],
781
+ ]) {
782
+ const projectDir = join(dir, id);
783
+ if (!existsSync(projectDir))
784
+ continue;
785
+ let entries;
786
+ try {
787
+ entries = readdirSync(projectDir);
788
+ }
789
+ catch {
790
+ continue;
791
+ }
792
+ for (const entry of entries) {
793
+ const ext = entry.slice(entry.lastIndexOf(".")).toLowerCase();
794
+ if (!imageExts.has(ext))
795
+ continue;
796
+ const fullPath = join(projectDir, entry);
797
+ try {
798
+ if (!statSync(fullPath).isFile())
799
+ continue;
800
+ }
801
+ catch {
802
+ continue;
803
+ }
804
+ results.push({
805
+ filename: entry,
806
+ url: `/api/files?path=${encodeURIComponent(fullPath)}`,
807
+ source,
808
+ });
809
+ }
810
+ }
811
+ // Sort newest first (by filename which often includes timestamp)
812
+ results.sort((a, b) => b.filename.localeCompare(a.filename));
813
+ res.json(results);
814
+ });
815
+ // List project videos (from generated outputs)
816
+ app.get("/api/projects/:id/videos", (req, res) => {
817
+ const { id } = req.params;
818
+ if (!isValidProjectId(id)) {
819
+ res.status(400).json({ error: "Invalid project ID" });
820
+ return;
821
+ }
822
+ const videoExts = new Set([".mp4"]);
823
+ const raw = [];
824
+ // Videos can live in both the output dir (generated) and the upload dir.
825
+ for (const dir of [OUTPUT_DIR, UPLOAD_DIR]) {
826
+ const projectDir = join(dir, id);
827
+ if (!existsSync(projectDir))
828
+ continue;
829
+ let entries;
830
+ try {
831
+ entries = readdirSync(projectDir);
832
+ }
833
+ catch {
834
+ continue;
835
+ }
836
+ for (const entry of entries) {
837
+ const ext = entry.slice(entry.lastIndexOf(".")).toLowerCase();
838
+ if (!videoExts.has(ext))
839
+ continue;
840
+ const fullPath = join(projectDir, entry);
841
+ let stats;
842
+ try {
843
+ stats = statSync(fullPath);
844
+ if (!stats.isFile())
845
+ continue;
846
+ }
847
+ catch {
848
+ continue;
849
+ }
850
+ raw.push({
851
+ filename: entry,
852
+ url: `/api/files?path=${encodeURIComponent(fullPath)}`,
853
+ birthtime: stats.birthtimeMs,
854
+ });
855
+ }
856
+ }
857
+ // Sort newest first by file creation date
858
+ raw.sort((a, b) => b.birthtime - a.birthtime);
859
+ const results = raw.map(({ filename, url }) => ({ filename, url }));
860
+ res.json(results);
861
+ });
862
+ // List project audio (from uploads)
863
+ app.get("/api/projects/:id/audios", (req, res) => {
864
+ const { id } = req.params;
865
+ if (!isValidProjectId(id)) {
866
+ res.status(400).json({ error: "Invalid project ID" });
867
+ return;
868
+ }
869
+ const audioExts = new Set([
870
+ ".mp3",
871
+ ".wav",
872
+ ".m4a",
873
+ ".flac",
874
+ ".aac",
875
+ ".ogg",
876
+ ".opus",
877
+ ]);
878
+ const raw = [];
879
+ for (const dir of [UPLOAD_DIR, OUTPUT_DIR]) {
880
+ const projectDir = join(dir, id);
881
+ if (!existsSync(projectDir))
882
+ continue;
883
+ let entries;
884
+ try {
885
+ entries = readdirSync(projectDir);
886
+ }
887
+ catch {
888
+ continue;
889
+ }
890
+ for (const entry of entries) {
891
+ const ext = entry.slice(entry.lastIndexOf(".")).toLowerCase();
892
+ if (!audioExts.has(ext))
893
+ continue;
894
+ const fullPath = join(projectDir, entry);
895
+ let stats;
896
+ try {
897
+ stats = statSync(fullPath);
898
+ if (!stats.isFile())
899
+ continue;
900
+ }
901
+ catch {
902
+ continue;
903
+ }
904
+ raw.push({
905
+ filename: entry,
906
+ url: `/api/files?path=${encodeURIComponent(fullPath)}`,
907
+ birthtime: stats.birthtimeMs,
908
+ });
909
+ }
910
+ }
911
+ raw.sort((a, b) => b.birthtime - a.birthtime);
912
+ res.json(raw.map(({ filename, url }) => ({ filename, url })));
913
+ });
914
+ // ========== Render: Text-to-Image ==========
915
+ app.post("/api/render/text-to-image", async (req, res) => {
916
+ const { prompt, projectId, width = 512, height = 512, device = "mps", } = req.body || {};
917
+ if (!prompt) {
918
+ res.status(400).json({ error: "Prompt is required" });
919
+ return;
920
+ }
921
+ if (!projectId) {
922
+ res.status(400).json({ error: "Project ID is required" });
923
+ return;
924
+ }
925
+ // z-image-mps requires width/height to be multiples of 16. Snap any
926
+ // client-provided size (e.g. an aspect-ratio-derived height) to the
927
+ // nearest valid multiple so generation does not fail.
928
+ const imageWidth = Math.max(16, Math.round(Number(width) / 16) * 16);
929
+ const imageHeight = Math.max(16, Math.round(Number(height) / 16) * 16);
930
+ // SSE headers
931
+ res.writeHead(200, {
932
+ "Content-Type": "text/event-stream",
933
+ "Cache-Control": "no-cache",
934
+ Connection: "keep-alive",
935
+ "X-Accel-Buffering": "no",
936
+ });
937
+ const send = (event, data) => {
938
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
939
+ };
940
+ try {
941
+ const zImageFolder = join(PYTHON_DIR, "z-image-mps");
942
+ if (!existsSync(zImageFolder)) {
943
+ send("error", { error: "z-image-mps not found. Run setup first." });
944
+ res.end();
945
+ return;
946
+ }
947
+ const uvPath = await getUvPath();
948
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
949
+ ensureDir(projectOutputDir);
950
+ const outputFile = `img-${Date.now()}.png`;
951
+ const outputPath = join(projectOutputDir, outputFile);
952
+ send("progress", {
953
+ status: "starting",
954
+ label: "Generating image...",
955
+ outputFile,
956
+ });
957
+ const proc = spawn([
958
+ uvPath,
959
+ "run",
960
+ "z-image-mps.py",
961
+ "-p",
962
+ prompt,
963
+ "--height",
964
+ String(imageHeight),
965
+ "--width",
966
+ String(imageWidth),
967
+ "--output",
968
+ outputPath,
969
+ "--device",
970
+ device,
971
+ ], {
972
+ cwd: zImageFolder,
973
+ stdout: "pipe",
974
+ stderr: "pipe",
975
+ });
976
+ activeProc = proc;
977
+ // Stream stdout/stderr concurrently
978
+ const stdoutPromise = streamToSSE(proc.stdout, "Image", send);
979
+ const stderrText = await streamToSSE(proc.stderr, "Image", send);
980
+ await stdoutPromise;
981
+ const exitCode = await proc.exited;
982
+ const success = exitCode === 0 && existsSync(outputPath);
983
+ if (success) {
984
+ send("complete", {
985
+ success: true,
986
+ path: outputPath,
987
+ filename: outputFile,
988
+ });
989
+ }
990
+ else {
991
+ send("error", {
992
+ error: stderrText || `Process exited with code ${exitCode}`,
993
+ exitCode,
994
+ });
995
+ }
996
+ }
997
+ catch (e) {
998
+ send("error", { error: String(e) });
999
+ }
1000
+ finally {
1001
+ activeProc = null;
1002
+ res.end();
1003
+ }
1004
+ });
1005
+ // ========== Render: Image-to-Video ==========
1006
+ app.post("/api/render/image-to-video", async (req, res) => {
1007
+ const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", } = req.body || {};
1008
+ if (!prompt) {
1009
+ res.status(400).json({ error: "Prompt is required" });
1010
+ return;
1011
+ }
1012
+ if (!imagePath) {
1013
+ res.status(400).json({ error: "Image path is required" });
1014
+ return;
1015
+ }
1016
+ if (!projectId || !isValidProjectId(String(projectId))) {
1017
+ res.status(400).json({ error: "Invalid project ID" });
1018
+ return;
1019
+ }
1020
+ if (typeof mode !== "string" ||
1021
+ !Object.keys(VIDEO_STAGE_FLAGS).includes(mode)) {
1022
+ res.status(400).json({ error: "Invalid mode" });
1023
+ return;
1024
+ }
1025
+ // Resolve image path — only allow project-relative paths (no absolute paths)
1026
+ const resolvedImage = resolveSafePath(imagePath, projectId);
1027
+ if (!resolvedImage) {
1028
+ res.status(400).json({
1029
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
1030
+ });
1031
+ return;
1032
+ }
1033
+ // SSE headers
1034
+ res.writeHead(200, {
1035
+ "Content-Type": "text/event-stream",
1036
+ "Cache-Control": "no-cache",
1037
+ Connection: "keep-alive",
1038
+ "X-Accel-Buffering": "no",
1039
+ });
1040
+ const send = (event, data) => {
1041
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1042
+ };
1043
+ try {
1044
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
1045
+ if (!existsSync(ltxFolder)) {
1046
+ send("error", { error: "ltx-2-mlx not found. Run setup first." });
1047
+ res.end();
1048
+ return;
1049
+ }
1050
+ const uvPath = await getUvPath();
1051
+ const projectOutputDir = resolveOutputDir(outputDir, projectId);
1052
+ if (!projectOutputDir) {
1053
+ send("error", { error: "Invalid output directory." });
1054
+ res.end();
1055
+ return;
1056
+ }
1057
+ const outputFile = `video-${Date.now()}.mp4`;
1058
+ const outputPath = join(projectOutputDir, outputFile);
1059
+ const videoWidth = Number(width) || 480;
1060
+ const videoHeight = Number(height) || 480;
1061
+ const videoFrames = Number(frames) || 121;
1062
+ const videoFps = Number(frameRate) || 24;
1063
+ send("progress", {
1064
+ status: "starting",
1065
+ label: "Generating video...",
1066
+ outputFile,
1067
+ settings: {
1068
+ width: videoWidth,
1069
+ height: videoHeight,
1070
+ frames: videoFrames,
1071
+ fps: videoFps,
1072
+ },
1073
+ });
1074
+ const proc = spawn([
1075
+ uvPath,
1076
+ "run",
1077
+ "ltx-2-mlx",
1078
+ "generate",
1079
+ "--model",
1080
+ "dgrauet/ltx-2.3-mlx-q8",
1081
+ "--prompt",
1082
+ prompt,
1083
+ stageFlagFor(mode),
1084
+ // "--low-ram",
1085
+ "--frames",
1086
+ String(videoFrames),
1087
+ "--width",
1088
+ String(videoWidth),
1089
+ "--height",
1090
+ String(videoHeight),
1091
+ "--frame-rate",
1092
+ String(videoFps),
1093
+ "--image",
1094
+ resolvedImage,
1095
+ "--output",
1096
+ outputPath,
1097
+ ], {
1098
+ env: process.env,
1099
+ cwd: ltxFolder,
1100
+ stdout: "pipe",
1101
+ stderr: "pipe",
1102
+ });
1103
+ activeProc = proc;
1104
+ // Stream stdout/stderr concurrently
1105
+ const stdoutPromise = streamToSSE(proc.stdout, "Video", send);
1106
+ const stderrText = await streamToSSE(proc.stderr, "Video", send);
1107
+ await stdoutPromise;
1108
+ const exitCode = await proc.exited;
1109
+ const success = exitCode === 0 && existsSync(outputPath);
1110
+ if (success) {
1111
+ send("complete", {
1112
+ success: true,
1113
+ path: outputPath,
1114
+ filename: outputFile,
1115
+ });
1116
+ }
1117
+ else {
1118
+ send("error", {
1119
+ error: stderrText || `Process exited with code ${exitCode}`,
1120
+ exitCode,
1121
+ });
1122
+ }
1123
+ }
1124
+ catch (e) {
1125
+ send("error", { error: String(e) });
1126
+ }
1127
+ finally {
1128
+ activeProc = null;
1129
+ res.end();
1130
+ }
1131
+ });
1132
+ // ========== Movie Studio: Render (characters/places/scenes → images/videos) ==========
1133
+ app.post("/api/movie-studio/render", async (req, res) => {
1134
+ const { projectId, characters, places, scenes } = req.body || {};
1135
+ if (!projectId || !isValidProjectId(String(projectId))) {
1136
+ res.status(400).json({ error: "Invalid project ID" });
1137
+ return;
1138
+ }
1139
+ if (!Array.isArray(characters) ||
1140
+ !Array.isArray(places) ||
1141
+ !Array.isArray(scenes)) {
1142
+ res
1143
+ .status(400)
1144
+ .json({ error: "characters, places, scenes are required" });
1145
+ return;
1146
+ }
1147
+ res.writeHead(200, {
1148
+ "Content-Type": "text/event-stream",
1149
+ "Cache-Control": "no-cache",
1150
+ Connection: "keep-alive",
1151
+ "X-Accel-Buffering": "no",
1152
+ });
1153
+ const send = (event, data) => {
1154
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1155
+ };
1156
+ const outputDir = join(OUTPUT_DIR, String(projectId));
1157
+ ensureDir(outputDir);
1158
+ let stepCount = 0;
1159
+ const totalSteps = (Array.isArray(characters) ? characters.length : 0) +
1160
+ (Array.isArray(places) ? places.length : 0) +
1161
+ (Array.isArray(scenes) ? scenes.length : 0) * 2;
1162
+ const progress = (label) => {
1163
+ stepCount += 1;
1164
+ send("progress", { label, current: stepCount, total: totalSteps });
1165
+ };
1166
+ const runStep = async (args, opts) => {
1167
+ const proc = spawn(args, {
1168
+ env: process.env,
1169
+ stdout: "pipe",
1170
+ stderr: "pipe",
1171
+ ...(opts.cwd ? { cwd: opts.cwd } : {}),
1172
+ });
1173
+ activeProc = proc;
1174
+ const stdoutPromise = streamToSSE(proc.stdout, opts.label, send);
1175
+ const stderrText = await streamToSSE(proc.stderr, opts.label, send);
1176
+ await stdoutPromise;
1177
+ const exitCode = await proc.exited;
1178
+ const success = exitCode === 0 && (!opts.outputPath || existsSync(opts.outputPath));
1179
+ return { success, stderr: stderrText };
1180
+ };
1181
+ const slug = (v) => String(v || "")
1182
+ .trim()
1183
+ .replace(/[^a-zA-Z0-9_-]/g, "_");
1184
+ try {
1185
+ const mlxgen = await getMlxgenBin();
1186
+ // 1. Character images (text-to-image).
1187
+ for (const c of characters) {
1188
+ const s = slug(c?.slug);
1189
+ const prompt = String(c?.imagePrompt || "").trim();
1190
+ if (!s || !prompt)
1191
+ continue;
1192
+ progress(`Generating character: ${c?.name || s}`);
1193
+ const outputFile = `character-${s}.png`;
1194
+ const outputPath = join(outputDir, outputFile);
1195
+ const result = await runStep([
1196
+ mlxgen,
1197
+ "generate",
1198
+ "--model",
1199
+ Z_IMAGE_MODEL,
1200
+ "--prompt",
1201
+ prompt,
1202
+ "--output",
1203
+ outputPath,
1204
+ "--steps",
1205
+ "6",
1206
+ "--width",
1207
+ "1024",
1208
+ "--height",
1209
+ "1024",
1210
+ ], { label: "Character", outputPath });
1211
+ if (!result.success) {
1212
+ send("error", { error: `Character "${s}" failed: ${result.stderr}` });
1213
+ return;
1214
+ }
1215
+ send("image", {
1216
+ kind: "character",
1217
+ slug: s,
1218
+ filename: outputFile,
1219
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
1220
+ });
1221
+ }
1222
+ // 2. Place images (text-to-image).
1223
+ for (const p of places) {
1224
+ const s = slug(p?.slug);
1225
+ const prompt = String(p?.imagePrompt || "").trim();
1226
+ if (!s || !prompt)
1227
+ continue;
1228
+ progress(`Generating place: ${p?.name || s}`);
1229
+ const outputFile = `place-${s}.png`;
1230
+ const outputPath = join(outputDir, outputFile);
1231
+ const result = await runStep([
1232
+ mlxgen,
1233
+ "generate",
1234
+ "--model",
1235
+ Z_IMAGE_MODEL,
1236
+ "--prompt",
1237
+ prompt,
1238
+ "--output",
1239
+ outputPath,
1240
+ "--steps",
1241
+ "6",
1242
+ "--width",
1243
+ "1024",
1244
+ "--height",
1245
+ "1024",
1246
+ ], { label: "Place", outputPath });
1247
+ if (!result.success) {
1248
+ send("error", { error: `Place "${s}" failed: ${result.stderr}` });
1249
+ return;
1250
+ }
1251
+ send("image", {
1252
+ kind: "place",
1253
+ slug: s,
1254
+ filename: outputFile,
1255
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
1256
+ });
1257
+ }
1258
+ // 3. Scenes: image (fast-image-edit) then video (ltx-2.3).
1259
+ const uvPath = await getUvPath();
1260
+ const sceneResults = [];
1261
+ for (const sc of scenes) {
1262
+ const s = slug(sc?.slug);
1263
+ if (!s)
1264
+ continue;
1265
+ // 3a. Scene image via fast-image-edit (FLUX.2 Klein).
1266
+ progress(`Generating scene image: ${s}`);
1267
+ const image = await generateSceneImage(String(projectId), sc);
1268
+ if ("error" in image) {
1269
+ send("error", {
1270
+ error: `Scene image "${s}" failed: ${image.error}`,
1271
+ });
1272
+ continue;
1273
+ }
1274
+ send("image", {
1275
+ kind: "scene",
1276
+ slug: s,
1277
+ filename: image.filename,
1278
+ url: image.url,
1279
+ });
1280
+ // 3b. Scene video via LTX-2.3 (320p, 9:16, distilled).
1281
+ progress(`Generating scene video: ${s}`);
1282
+ const video = await generateSceneVideo(uvPath, String(projectId), sc, characters);
1283
+ if ("error" in video) {
1284
+ send("error", {
1285
+ error: `Scene video "${s}" failed: ${video.error}`,
1286
+ });
1287
+ continue;
1288
+ }
1289
+ send("video", {
1290
+ slug: s,
1291
+ filename: video.filename,
1292
+ url: video.url,
1293
+ });
1294
+ sceneResults.push({
1295
+ slug: s,
1296
+ image: image.filename,
1297
+ video: video.filename,
1298
+ });
1299
+ }
1300
+ send("complete", { scenes: sceneResults });
1301
+ }
1302
+ catch (e) {
1303
+ send("error", { error: String(e) });
1304
+ }
1305
+ finally {
1306
+ activeProc = null;
1307
+ res.end();
1308
+ }
1309
+ });
1310
+ // ========== Movie Studio: Render Assets (characters + places images) ==========
1311
+ app.post("/api/movie-studio/render-assets", async (req, res) => {
1312
+ const { projectId, characters, places } = req.body || {};
1313
+ if (!projectId || !isValidProjectId(String(projectId))) {
1314
+ res.status(400).json({ error: "Invalid project ID" });
1315
+ return;
1316
+ }
1317
+ if (!Array.isArray(characters) || !Array.isArray(places)) {
1318
+ res.status(400).json({ error: "characters and places are required" });
1319
+ return;
1320
+ }
1321
+ res.writeHead(200, {
1322
+ "Content-Type": "text/event-stream",
1323
+ "Cache-Control": "no-cache",
1324
+ Connection: "keep-alive",
1325
+ "X-Accel-Buffering": "no",
1326
+ });
1327
+ const send = (event, data) => {
1328
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1329
+ };
1330
+ const slugify = (v) => String(v || "")
1331
+ .trim()
1332
+ .replace(/[^a-zA-Z0-9_-]/g, "_");
1333
+ const assets = { characters: [], places: [] };
1334
+ try {
1335
+ for (const c of characters) {
1336
+ const slug = slugify(c?.slug);
1337
+ const prompt = String(c?.imagePrompt || "").trim();
1338
+ if (!slug || !prompt)
1339
+ continue;
1340
+ send("progress", { label: `Generating character: ${c?.name || slug}` });
1341
+ const r = await generateAssetImage(String(projectId), "character", slug, prompt);
1342
+ if ("error" in r) {
1343
+ send("error", { error: r.error });
1344
+ continue;
1345
+ }
1346
+ assets.characters.push({ slug, ...r });
1347
+ send("image", { kind: "character", slug, ...r });
1348
+ }
1349
+ for (const p of places) {
1350
+ const slug = slugify(p?.slug);
1351
+ const prompt = String(p?.imagePrompt || "").trim();
1352
+ if (!slug || !prompt)
1353
+ continue;
1354
+ send("progress", { label: `Generating place: ${p?.name || slug}` });
1355
+ const r = await generateAssetImage(String(projectId), "place", slug, prompt);
1356
+ if ("error" in r) {
1357
+ send("error", { error: r.error });
1358
+ continue;
1359
+ }
1360
+ assets.places.push({ slug, ...r });
1361
+ send("image", { kind: "place", slug, ...r });
1362
+ }
1363
+ send("complete", { assets });
1364
+ }
1365
+ catch (e) {
1366
+ send("error", { error: String(e) });
1367
+ }
1368
+ finally {
1369
+ activeProc = null;
1370
+ res.end();
1371
+ }
1372
+ });
1373
+ // Regenerate a single character/place image.
1374
+ app.post("/api/movie-studio/render-asset", async (req, res) => {
1375
+ const { projectId, kind, slug, prompt } = req.body || {};
1376
+ if (!projectId || !isValidProjectId(String(projectId))) {
1377
+ res.status(400).json({ error: "Invalid project ID" });
1378
+ return;
1379
+ }
1380
+ if (kind !== "character" && kind !== "place") {
1381
+ res.status(400).json({ error: "kind must be 'character' or 'place'" });
1382
+ return;
1383
+ }
1384
+ const safeSlug = String(slug || "")
1385
+ .trim()
1386
+ .replace(/[^a-zA-Z0-9_-]/g, "_");
1387
+ const safePrompt = String(prompt || "").trim();
1388
+ if (!safeSlug || !safePrompt) {
1389
+ res.status(400).json({ error: "slug and prompt are required" });
1390
+ return;
1391
+ }
1392
+ try {
1393
+ const r = await generateAssetImage(String(projectId), kind, safeSlug, safePrompt);
1394
+ if ("error" in r) {
1395
+ res.status(500).json({ error: r.error });
1396
+ return;
1397
+ }
1398
+ res.json(r);
1399
+ }
1400
+ catch (e) {
1401
+ res.status(500).json({ error: String(e) });
1402
+ }
1403
+ finally {
1404
+ activeProc = null;
1405
+ }
1406
+ });
1407
+ // ========== Movie Studio: Render Videos (scene videos only) ==========
1408
+ app.post("/api/movie-studio/render-videos", async (req, res) => {
1409
+ const { projectId, characters, scenes } = req.body || {};
1410
+ if (!projectId || !isValidProjectId(String(projectId))) {
1411
+ res.status(400).json({ error: "Invalid project ID" });
1412
+ return;
1413
+ }
1414
+ if (!Array.isArray(characters) || !Array.isArray(scenes)) {
1415
+ res.status(400).json({ error: "characters and scenes are required" });
1416
+ return;
1417
+ }
1418
+ res.writeHead(200, {
1419
+ "Content-Type": "text/event-stream",
1420
+ "Cache-Control": "no-cache",
1421
+ Connection: "keep-alive",
1422
+ "X-Accel-Buffering": "no",
1423
+ });
1424
+ const send = (event, data) => {
1425
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1426
+ };
1427
+ const total = scenes.length;
1428
+ let current = 0;
1429
+ try {
1430
+ const uvPath = await getUvPath();
1431
+ const videos = [];
1432
+ for (const sc of scenes) {
1433
+ const slug = String(sc?.slug || "")
1434
+ .trim()
1435
+ .replace(/[^a-zA-Z0-9_-]/g, "_");
1436
+ if (!slug)
1437
+ continue;
1438
+ current += 1;
1439
+ send("progress", {
1440
+ label: `Generating video: ${slug}`,
1441
+ current,
1442
+ total,
1443
+ });
1444
+ const r = await generateSceneVideo(uvPath, String(projectId), sc, characters);
1445
+ if ("error" in r) {
1446
+ send("error", { error: r.error });
1447
+ continue;
1448
+ }
1449
+ videos.push({ slug, ...r });
1450
+ send("video", { slug, ...r });
1451
+ }
1452
+ send("complete", { videos });
1453
+ }
1454
+ catch (e) {
1455
+ send("error", { error: String(e) });
1456
+ }
1457
+ finally {
1458
+ activeProc = null;
1459
+ res.end();
1460
+ }
1461
+ });
1462
+ // Regenerate a single scene video.
1463
+ app.post("/api/movie-studio/render-video", async (req, res) => {
1464
+ const { projectId, scene, characters } = req.body || {};
1465
+ if (!projectId || !isValidProjectId(String(projectId))) {
1466
+ res.status(400).json({ error: "Invalid project ID" });
1467
+ return;
1468
+ }
1469
+ if (!scene || typeof scene !== "object") {
1470
+ res.status(400).json({ error: "scene is required" });
1471
+ return;
1472
+ }
1473
+ try {
1474
+ const uvPath = await getUvPath();
1475
+ const r = await generateSceneVideo(uvPath, String(projectId), scene, Array.isArray(characters) ? characters : []);
1476
+ if ("error" in r) {
1477
+ res.status(500).json({ error: r.error });
1478
+ return;
1479
+ }
1480
+ res.json(r);
1481
+ }
1482
+ catch (e) {
1483
+ res.status(500).json({ error: String(e) });
1484
+ }
1485
+ finally {
1486
+ activeProc = null;
1487
+ }
1488
+ });
1489
+ // ========== Movie Studio: Render Scene Images (fast-image-edit) ==========
1490
+ app.post("/api/movie-studio/render-scene-images", async (req, res) => {
1491
+ const { projectId, scenes } = req.body || {};
1492
+ if (!projectId || !isValidProjectId(String(projectId))) {
1493
+ res.status(400).json({ error: "Invalid project ID" });
1494
+ return;
1495
+ }
1496
+ if (!Array.isArray(scenes)) {
1497
+ res.status(400).json({ error: "scenes are required" });
1498
+ return;
1499
+ }
1500
+ res.writeHead(200, {
1501
+ "Content-Type": "text/event-stream",
1502
+ "Cache-Control": "no-cache",
1503
+ Connection: "keep-alive",
1504
+ "X-Accel-Buffering": "no",
1505
+ });
1506
+ const send = (event, data) => {
1507
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1508
+ };
1509
+ const total = scenes.length;
1510
+ let current = 0;
1511
+ try {
1512
+ const images = [];
1513
+ for (const sc of scenes) {
1514
+ const slug = slugify(sc?.slug);
1515
+ if (!slug)
1516
+ continue;
1517
+ current += 1;
1518
+ send("progress", {
1519
+ label: `Generating scene image: ${slug}`,
1520
+ current,
1521
+ total,
1522
+ });
1523
+ const r = await generateSceneImage(String(projectId), sc);
1524
+ if ("error" in r) {
1525
+ send("error", { error: r.error });
1526
+ continue;
1527
+ }
1528
+ images.push({ slug, ...r });
1529
+ send("image", { slug, ...r });
1530
+ }
1531
+ send("complete", { images });
1532
+ }
1533
+ catch (e) {
1534
+ send("error", { error: String(e) });
1535
+ }
1536
+ finally {
1537
+ activeProc = null;
1538
+ res.end();
1539
+ }
1540
+ });
1541
+ // Regenerate a single scene image.
1542
+ app.post("/api/movie-studio/render-scene-image", async (req, res) => {
1543
+ const { projectId, scene } = req.body || {};
1544
+ if (!projectId || !isValidProjectId(String(projectId))) {
1545
+ res.status(400).json({ error: "Invalid project ID" });
1546
+ return;
1547
+ }
1548
+ if (!scene || typeof scene !== "object") {
1549
+ res.status(400).json({ error: "scene is required" });
1550
+ return;
1551
+ }
1552
+ try {
1553
+ const r = await generateSceneImage(String(projectId), scene);
1554
+ if ("error" in r) {
1555
+ res.status(500).json({ error: r.error });
1556
+ return;
1557
+ }
1558
+ res.json(r);
1559
+ }
1560
+ catch (e) {
1561
+ res.status(500).json({ error: String(e) });
1562
+ }
1563
+ finally {
1564
+ activeProc = null;
1565
+ }
1566
+ });
1567
+ // ========== Render: Text-to-Video ==========
1568
+ app.post("/api/render/text-to-video", async (req, res) => {
1569
+ const { prompt, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", } = req.body || {};
1570
+ if (!prompt) {
1571
+ res.status(400).json({ error: "Prompt is required" });
1572
+ return;
1573
+ }
1574
+ if (!projectId || !isValidProjectId(String(projectId))) {
1575
+ res.status(400).json({ error: "Invalid project ID" });
1576
+ return;
1577
+ }
1578
+ if (typeof mode !== "string" ||
1579
+ !Object.keys(VIDEO_STAGE_FLAGS).includes(mode)) {
1580
+ res.status(400).json({ error: "Invalid mode" });
1581
+ return;
1582
+ }
1583
+ // SSE headers
1584
+ res.writeHead(200, {
1585
+ "Content-Type": "text/event-stream",
1586
+ "Cache-Control": "no-cache",
1587
+ Connection: "keep-alive",
1588
+ "X-Accel-Buffering": "no",
1589
+ });
1590
+ const send = (event, data) => {
1591
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1592
+ };
1593
+ try {
1594
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
1595
+ if (!existsSync(ltxFolder)) {
1596
+ send("error", { error: "ltx-2-mlx not found. Run setup first." });
1597
+ res.end();
1598
+ return;
1599
+ }
1600
+ const uvPath = await getUvPath();
1601
+ const projectOutputDir = resolveOutputDir(outputDir, projectId);
1602
+ if (!projectOutputDir) {
1603
+ send("error", { error: "Invalid output directory." });
1604
+ res.end();
1605
+ return;
1606
+ }
1607
+ const outputFile = `video-${Date.now()}.mp4`;
1608
+ const outputPath = join(projectOutputDir, outputFile);
1609
+ const videoWidth = Number(width) || 480;
1610
+ const videoHeight = Number(height) || 480;
1611
+ const videoFrames = Number(frames) || 121;
1612
+ const videoFps = Number(frameRate) || 24;
1613
+ send("progress", {
1614
+ status: "starting",
1615
+ label: "Generating video...",
1616
+ outputFile,
1617
+ settings: {
1618
+ width: videoWidth,
1619
+ height: videoHeight,
1620
+ frames: videoFrames,
1621
+ fps: videoFps,
1622
+ },
1623
+ });
1624
+ const proc = spawn([
1625
+ uvPath,
1626
+ "run",
1627
+ "ltx-2-mlx",
1628
+ "generate",
1629
+ "--model",
1630
+ "dgrauet/ltx-2.3-mlx-q8",
1631
+ "--prompt",
1632
+ prompt,
1633
+ stageFlagFor(mode),
1634
+ // "--low-ram",
1635
+ "--frames",
1636
+ String(videoFrames),
1637
+ "--width",
1638
+ String(videoWidth),
1639
+ "--height",
1640
+ String(videoHeight),
1641
+ "--frame-rate",
1642
+ String(videoFps),
1643
+ "--output",
1644
+ outputPath,
1645
+ ], {
1646
+ env: process.env,
1647
+ cwd: ltxFolder,
1648
+ stdout: "pipe",
1649
+ stderr: "pipe",
1650
+ });
1651
+ activeProc = proc;
1652
+ const stdoutPromise = streamToSSE(proc.stdout, "TextVideo", send);
1653
+ const stderrText = await streamToSSE(proc.stderr, "TextVideo", send);
1654
+ await stdoutPromise;
1655
+ const exitCode = await proc.exited;
1656
+ const success = exitCode === 0 && existsSync(outputPath);
1657
+ if (success) {
1658
+ send("complete", {
1659
+ success: true,
1660
+ path: outputPath,
1661
+ filename: outputFile,
1662
+ });
1663
+ }
1664
+ else {
1665
+ send("error", {
1666
+ error: stderrText || `Process exited with code ${exitCode}`,
1667
+ exitCode,
1668
+ });
1669
+ }
1670
+ }
1671
+ catch (e) {
1672
+ send("error", { error: String(e) });
1673
+ }
1674
+ finally {
1675
+ activeProc = null;
1676
+ res.end();
1677
+ }
1678
+ });
1679
+ // ========== Render: Extend-Video ==========
1680
+ app.post("/api/render/extend-video", async (req, res) => {
1681
+ const { prompt, videoPath, projectId, extendFrames = 2 } = req.body || {};
1682
+ if (!prompt) {
1683
+ res.status(400).json({ error: "Prompt is required" });
1684
+ return;
1685
+ }
1686
+ if (!videoPath) {
1687
+ res.status(400).json({ error: "Video path is required" });
1688
+ return;
1689
+ }
1690
+ if (!projectId) {
1691
+ res.status(400).json({ error: "Project ID is required" });
1692
+ return;
1693
+ }
1694
+ // Resolve video path — only bare .mp4 filenames in this project's output dir
1695
+ const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
1696
+ if (!resolvedVideo) {
1697
+ res.status(400).json({
1698
+ error: "Invalid video path. Provide a filename previously generated in this project.",
1699
+ });
1700
+ return;
1701
+ }
1702
+ // SSE headers
1703
+ res.writeHead(200, {
1704
+ "Content-Type": "text/event-stream",
1705
+ "Cache-Control": "no-cache",
1706
+ Connection: "keep-alive",
1707
+ "X-Accel-Buffering": "no",
1708
+ });
1709
+ const send = (event, data) => {
1710
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1711
+ };
1712
+ try {
1713
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
1714
+ if (!existsSync(ltxFolder)) {
1715
+ send("error", { error: "ltx-2-mlx not found. Run setup first." });
1716
+ res.end();
1717
+ return;
1718
+ }
1719
+ const uvPath = await getUvPath();
1720
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
1721
+ ensureDir(projectOutputDir);
1722
+ const outputFile = `extended-${Date.now()}.mp4`;
1723
+ const outputPath = join(projectOutputDir, outputFile);
1724
+ const framesToAdd = Number(extendFrames) || 2;
1725
+ send("progress", {
1726
+ status: "starting",
1727
+ label: "Extending video...",
1728
+ inputFile: resolvedVideo,
1729
+ outputFile,
1730
+ settings: { extendFrames: framesToAdd },
1731
+ });
1732
+ const proc = spawn([
1733
+ uvPath,
1734
+ "run",
1735
+ "ltx-2-mlx",
1736
+ "extend",
1737
+ "--model",
1738
+ "dgrauet/ltx-2.3-mlx-q8",
1739
+ "--prompt",
1740
+ prompt,
1741
+ "--video",
1742
+ resolvedVideo,
1743
+ "--extend-frames",
1744
+ String(framesToAdd),
1745
+ "--output",
1746
+ outputPath,
1747
+ ], {
1748
+ env: process.env,
1749
+ cwd: ltxFolder,
1750
+ stdout: "pipe",
1751
+ stderr: "pipe",
1752
+ });
1753
+ activeProc = proc;
1754
+ // Stream stdout/stderr concurrently
1755
+ const stdoutPromise = streamToSSE(proc.stdout, "Extend", send);
1756
+ const stderrText = await streamToSSE(proc.stderr, "Extend", send);
1757
+ await stdoutPromise;
1758
+ const exitCode = await proc.exited;
1759
+ const success = exitCode === 0 && existsSync(outputPath);
1760
+ if (success) {
1761
+ send("complete", {
1762
+ success: true,
1763
+ path: outputPath,
1764
+ filename: outputFile,
1765
+ });
1766
+ }
1767
+ else {
1768
+ send("error", {
1769
+ error: stderrText || `Process exited with code ${exitCode}`,
1770
+ exitCode,
1771
+ });
1772
+ }
1773
+ }
1774
+ catch (e) {
1775
+ send("error", { error: String(e) });
1776
+ }
1777
+ finally {
1778
+ activeProc = null;
1779
+ res.end();
1780
+ }
1781
+ });
1782
+ // ========== MLX-Audio: Status ==========
1783
+ app.get("/api/mlxaudio/status", async (_req, res) => {
1784
+ // Never throw on a missing uv — installation state is based on the folder.
1785
+ try {
1786
+ await getUvPath();
1787
+ }
1788
+ catch {
1789
+ // uv not found; installed is still reported from the folder below.
1790
+ }
1791
+ res.json({
1792
+ installed: existsSync(join(PYTHON_DIR, "mlx-audio")),
1793
+ });
1794
+ });
1795
+ // ========== Render: Text-to-Speech (mlx-audio) ==========
1796
+ app.post("/api/render/tts", async (req, res) => {
1797
+ const { text, refAudioPath, projectId, outputDir, quality, voiceId } = req.body || {};
1798
+ if (!text || typeof text !== "string" || !text.trim()) {
1799
+ res.status(400).json({ error: "Text is required" });
1800
+ return;
1801
+ }
1802
+ if (!refAudioPath) {
1803
+ res.status(400).json({ error: "Reference audio is required" });
1804
+ return;
1805
+ }
1806
+ if (!projectId || !isValidProjectId(String(projectId))) {
1807
+ res.status(400).json({ error: "Invalid project ID" });
1808
+ return;
1809
+ }
1810
+ if (quality !== "low" && quality !== "high") {
1811
+ res.status(400).json({ error: "Quality must be 'low' or 'high'" });
1812
+ return;
1813
+ }
1814
+ // Resolve reference audio — only bare filenames in this project's dirs
1815
+ const resolvedRef = resolveSafePath(refAudioPath, projectId);
1816
+ if (!resolvedRef) {
1817
+ res.status(400).json({
1818
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
1819
+ });
1820
+ return;
1821
+ }
1822
+ // SSE headers
1823
+ res.writeHead(200, {
1824
+ "Content-Type": "text/event-stream",
1825
+ "Cache-Control": "no-cache",
1826
+ Connection: "keep-alive",
1827
+ "X-Accel-Buffering": "no",
1828
+ });
1829
+ const send = (event, data) => {
1830
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1831
+ };
1832
+ try {
1833
+ const uvPath = await getUvPath();
1834
+ const projectOutputDir = resolveOutputDir(outputDir, projectId);
1835
+ if (!projectOutputDir) {
1836
+ send("error", { error: "Invalid output directory." });
1837
+ res.end();
1838
+ return;
1839
+ }
1840
+ // TTS audio is saved under <projectOutputDir>/voices/<voiceId>/ so each
1841
+ // row's voiceover stays isolated in its own folder. Sanitize voiceId to
1842
+ // alphanumerics/_/- so it can never escape the voices directory.
1843
+ const safeVoiceId = String(voiceId ?? `voice-${Date.now()}`)
1844
+ .replace(/[^a-zA-Z0-9_-]/g, "_")
1845
+ .slice(0, 64);
1846
+ const voiceDir = join(projectOutputDir, "voices", safeVoiceId);
1847
+ ensureDir(voiceDir);
1848
+ // Regenerating a row must overwrite the previous clip: delete any
1849
+ // existing output so the generator writes a fresh audio_000.mp3.
1850
+ const prevAudio = join(voiceDir, TTS_OUTPUT_FILENAME);
1851
+ if (existsSync(prevAudio)) {
1852
+ try {
1853
+ unlinkSync(prevAudio);
1854
+ }
1855
+ catch {
1856
+ // Ignore — the generator will overwrite the file regardless.
1857
+ }
1858
+ }
1859
+ send("progress", {
1860
+ status: "starting",
1861
+ label: "Generating speech...",
1862
+ model: TTS_MODELS[quality],
1863
+ });
1864
+ // No --play flag: this is a silent server-side batch generation. The mp3
1865
+ // lands in <projectOutputDir>/voices/<voiceId>/, which is servable via
1866
+ // /api/files and resolvable via resolveSafePath for the mux step.
1867
+ const proc = spawn([
1868
+ uvPath,
1869
+ "run",
1870
+ "mlx_audio.tts.generate",
1871
+ "--model",
1872
+ TTS_MODELS[quality],
1873
+ "--text",
1874
+ text,
1875
+ "--ref_audio",
1876
+ resolvedRef,
1877
+ "--output",
1878
+ voiceDir,
1879
+ "--audio_format",
1880
+ "mp3",
1881
+ "--play",
1882
+ "--instruct",
1883
+ "slow down speech",
1884
+ ], {
1885
+ env: process.env,
1886
+ cwd: voiceDir,
1887
+ stdout: "pipe",
1888
+ stderr: "pipe",
1889
+ });
1890
+ activeProc = proc;
1891
+ const stdoutPromise = streamToSSE(proc.stdout, "TTS", send);
1892
+ const stderrText = await streamToSSE(proc.stderr, "TTS", send);
1893
+ await stdoutPromise;
1894
+ const exitCode = await proc.exited;
1895
+ if (exitCode === 0) {
1896
+ const path = resolveAudioFile(voiceDir);
1897
+ if (path) {
1898
+ send("complete", {
1899
+ success: true,
1900
+ path,
1901
+ filename: path.split(sep).pop(),
1902
+ });
1903
+ }
1904
+ else {
1905
+ send("error", {
1906
+ error: "TTS completed but no audio file was produced",
1907
+ });
1908
+ }
1909
+ }
1910
+ else {
1911
+ send("error", {
1912
+ error: stderrText || `Process exited with code ${exitCode}`,
1913
+ exitCode,
1914
+ });
1915
+ }
1916
+ }
1917
+ catch (e) {
1918
+ send("error", { error: String(e) });
1919
+ }
1920
+ finally {
1921
+ activeProc = null;
1922
+ res.end();
1923
+ }
1924
+ });
1925
+ // ========== Render: Voice Chat (TTS with reference voice) ==========
1926
+ app.post("/api/render/voice-chat", async (req, res) => {
1927
+ const { text, refAudioPath, projectId } = req.body || {};
1928
+ if (!text || typeof text !== "string" || !text.trim()) {
1929
+ res.status(400).json({ error: "Text is required" });
1930
+ return;
1931
+ }
1932
+ if (!refAudioPath) {
1933
+ res.status(400).json({ error: "Reference audio is required" });
1934
+ return;
1935
+ }
1936
+ if (!projectId || !isValidProjectId(String(projectId))) {
1937
+ res.status(400).json({ error: "Invalid project ID" });
1938
+ return;
1939
+ }
1940
+ // Resolve reference audio — only bare filenames in this project's dirs.
1941
+ const resolvedRef = resolveSafePath(refAudioPath, String(projectId));
1942
+ if (!resolvedRef) {
1943
+ res.status(400).json({
1944
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
1945
+ });
1946
+ return;
1947
+ }
1948
+ // SSE headers
1949
+ res.writeHead(200, {
1950
+ "Content-Type": "text/event-stream",
1951
+ "Cache-Control": "no-cache",
1952
+ Connection: "keep-alive",
1953
+ "X-Accel-Buffering": "no",
1954
+ });
1955
+ const send = (event, data) => {
1956
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1957
+ };
1958
+ try {
1959
+ const uvPath = await getUvPath();
1960
+ const projectOutputDir = resolveOutputDir(null, String(projectId));
1961
+ if (!projectOutputDir) {
1962
+ send("error", { error: "Invalid output directory." });
1963
+ res.end();
1964
+ return;
1965
+ }
1966
+ // Isolate each utterance so a new response never overwrites a previous
1967
+ // one. mlx_audio writes the clip as <dir>/audio_000.mp3.
1968
+ const voiceDir = join(projectOutputDir, "voice-chat", `chat-${Date.now()}`);
1969
+ ensureDir(voiceDir);
1970
+ send("progress", {
1971
+ status: "starting",
1972
+ label: "Generating voice...",
1973
+ });
1974
+ const proc = spawn([
1975
+ uvPath,
1976
+ "run",
1977
+ "mlx_audio.tts.generate",
1978
+ "--model",
1979
+ TTS_MODELS.high,
1980
+ "--text",
1981
+ text.trim(),
1982
+ "--ref_audio",
1983
+ resolvedRef,
1984
+ // "--play",
1985
+ "--output",
1986
+ voiceDir,
1987
+ "--audio_format",
1988
+ "mp3",
1989
+ // "--stream",
1990
+ // "--save",
1991
+ "--instruct",
1992
+ "slow down",
1993
+ ], {
1994
+ env: process.env,
1995
+ cwd: voiceDir,
1996
+ stdout: "pipe",
1997
+ stderr: "pipe",
1998
+ });
1999
+ activeProc = proc;
2000
+ const stdoutPromise = streamToSSE(proc.stdout, "VoiceChat", send);
2001
+ const stderrText = await streamToSSE(proc.stderr, "VoiceChat", send);
2002
+ await stdoutPromise;
2003
+ const exitCode = await proc.exited;
2004
+ if (exitCode === 0) {
2005
+ const path = resolveAudioFile(voiceDir);
2006
+ if (path) {
2007
+ send("complete", {
2008
+ success: true,
2009
+ path,
2010
+ filename: path.split(sep).pop(),
2011
+ });
2012
+ }
2013
+ else {
2014
+ send("error", {
2015
+ error: "TTS completed but no audio file was produced",
2016
+ });
2017
+ }
2018
+ }
2019
+ else {
2020
+ send("error", {
2021
+ error: stderrText || `Process exited with code ${exitCode}`,
2022
+ exitCode,
2023
+ });
2024
+ }
2025
+ }
2026
+ catch (e) {
2027
+ send("error", { error: String(e) });
2028
+ }
2029
+ finally {
2030
+ activeProc = null;
2031
+ res.end();
2032
+ }
2033
+ });
2034
+ // ========== Render: Mux Video + Audio ==========
2035
+ app.post("/api/render/mux-audio", async (req, res) => {
2036
+ const { videoPath, audioPath, projectId, outputDir } = req.body || {};
2037
+ if (!videoPath) {
2038
+ res.status(400).json({ error: "Video path is required" });
2039
+ return;
2040
+ }
2041
+ if (!audioPath) {
2042
+ res.status(400).json({ error: "Audio path is required" });
2043
+ return;
2044
+ }
2045
+ if (!projectId || !isValidProjectId(String(projectId))) {
2046
+ res.status(400).json({ error: "Invalid project ID" });
2047
+ return;
2048
+ }
2049
+ // Resolve video path — only bare .mp4 filenames in this project's output dir
2050
+ const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
2051
+ if (!resolvedVideo) {
2052
+ res.status(400).json({
2053
+ error: "Invalid video path. Provide a filename previously generated in this project.",
2054
+ });
2055
+ return;
2056
+ }
2057
+ // Resolve audio path — only bare filenames in this project's dirs
2058
+ const resolvedAudio = resolveSafePath(audioPath, projectId);
2059
+ if (!resolvedAudio) {
2060
+ res.status(400).json({
2061
+ error: "Invalid audio path. Provide a filename previously generated or uploaded to this project.",
2062
+ });
2063
+ return;
2064
+ }
2065
+ // SSE headers
2066
+ res.writeHead(200, {
2067
+ "Content-Type": "text/event-stream",
2068
+ "Cache-Control": "no-cache",
2069
+ Connection: "keep-alive",
2070
+ "X-Accel-Buffering": "no",
2071
+ });
2072
+ const send = (event, data) => {
2073
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2074
+ };
2075
+ try {
2076
+ const ffmpegBin = await getFfmpegBin();
2077
+ const projectOutputDir = resolveOutputDir(outputDir, projectId);
2078
+ if (!projectOutputDir) {
2079
+ send("error", { error: "Invalid output directory." });
2080
+ res.end();
2081
+ return;
2082
+ }
2083
+ const finalFile = `voice-${Date.now()}.mp4`;
2084
+ const finalPath = join(projectOutputDir, finalFile);
2085
+ send("progress", {
2086
+ status: "starting",
2087
+ label: "Muxing video and audio...",
2088
+ outputFile: finalFile,
2089
+ });
2090
+ const proc = spawn([
2091
+ ffmpegBin,
2092
+ "-y",
2093
+ "-i",
2094
+ resolvedVideo,
2095
+ "-i",
2096
+ resolvedAudio,
2097
+ "-map",
2098
+ "0:v",
2099
+ "-map",
2100
+ "1:a",
2101
+ "-c:v",
2102
+ "copy",
2103
+ "-c:a",
2104
+ "aac",
2105
+ "-b:a",
2106
+ "192k",
2107
+ "-shortest",
2108
+ finalPath,
2109
+ ], {
2110
+ stdout: "pipe",
2111
+ stderr: "pipe",
2112
+ });
2113
+ activeProc = proc;
2114
+ const stdoutPromise = streamToSSE(proc.stdout, "Mux", send);
2115
+ const stderrText = await streamToSSE(proc.stderr, "Mux", send);
2116
+ await stdoutPromise;
2117
+ const exitCode = await proc.exited;
2118
+ if (exitCode === 0 && existsSync(finalPath)) {
2119
+ send("complete", {
2120
+ success: true,
2121
+ path: finalPath,
2122
+ filename: finalFile,
2123
+ });
2124
+ }
2125
+ else {
2126
+ send("error", {
2127
+ error: stderrText || `Process exited with code ${exitCode}`,
2128
+ exitCode,
2129
+ });
2130
+ }
2131
+ }
2132
+ catch (e) {
2133
+ send("error", { error: String(e) });
2134
+ }
2135
+ finally {
2136
+ activeProc = null;
2137
+ res.end();
2138
+ }
2139
+ });
2140
+ // ========== MLX-Gen: Status ==========
2141
+ app.get("/api/mlxgen/status", (_req, res) => {
2142
+ res.json({
2143
+ installed: isMlxgenInstalled(),
2144
+ modelDownloaded: isModelDownloaded(),
2145
+ zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
2146
+ fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
2147
+ });
2148
+ });
2149
+ // ========== MLX-Gen: Install ==========
2150
+ app.post("/api/mlxgen/install", async (_req, res) => {
2151
+ res.writeHead(200, {
2152
+ "Content-Type": "text/event-stream",
2153
+ "Cache-Control": "no-cache",
2154
+ Connection: "keep-alive",
2155
+ "X-Accel-Buffering": "no",
2156
+ });
2157
+ const send = (event, data) => {
2158
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2159
+ };
2160
+ try {
2161
+ const uvPath = await getUvPath();
2162
+ send("progress", {
2163
+ status: "starting",
2164
+ label: "Installing mlx-gen...",
2165
+ });
2166
+ const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
2167
+ stdout: "pipe",
2168
+ stderr: "pipe",
2169
+ });
2170
+ activeProc = proc;
2171
+ const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
2172
+ const stderrText = await streamToSSE(proc.stderr, "Install", send);
2173
+ await stdoutPromise;
2174
+ const exitCode = await proc.exited;
2175
+ if (exitCode === 0) {
2176
+ send("complete", { success: true });
2177
+ }
2178
+ else {
2179
+ send("error", {
2180
+ error: stderrText || `Process exited with code ${exitCode}`,
2181
+ exitCode,
2182
+ });
2183
+ }
2184
+ }
2185
+ catch (e) {
2186
+ send("error", { error: String(e) });
2187
+ }
2188
+ finally {
2189
+ activeProc = null;
2190
+ res.end();
2191
+ }
2192
+ });
2193
+ // ========== MLX-Gen: Download Model ==========
2194
+ app.post("/api/mlxgen/download-model", async (_req, res) => {
2195
+ res.writeHead(200, {
2196
+ "Content-Type": "text/event-stream",
2197
+ "Cache-Control": "no-cache",
2198
+ Connection: "keep-alive",
2199
+ "X-Accel-Buffering": "no",
2200
+ });
2201
+ const send = (event, data) => {
2202
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2203
+ };
2204
+ try {
2205
+ const mlxgen = await getMlxgenBin();
2206
+ send("progress", {
2207
+ status: "starting",
2208
+ label: `Downloading model ${MLXGEN_MODEL}...`,
2209
+ });
2210
+ const proc = spawn([mlxgen, "download", "--model", MLXGEN_MODEL], {
2211
+ stdout: "pipe",
2212
+ stderr: "pipe",
2213
+ });
2214
+ activeProc = proc;
2215
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2216
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2217
+ await stdoutPromise;
2218
+ const exitCode = await proc.exited;
2219
+ if (exitCode === 0) {
2220
+ send("complete", { success: true });
2221
+ }
2222
+ else {
2223
+ send("error", {
2224
+ error: stderrText || `Process exited with code ${exitCode}`,
2225
+ exitCode,
2226
+ });
2227
+ }
2228
+ }
2229
+ catch (e) {
2230
+ send("error", { error: String(e) });
2231
+ }
2232
+ finally {
2233
+ activeProc = null;
2234
+ res.end();
2235
+ }
2236
+ });
2237
+ // ========== MLX-Gen: Download Z-Image Model ==========
2238
+ app.post("/api/mlxgen/download-z-model", async (_req, res) => {
2239
+ res.writeHead(200, {
2240
+ "Content-Type": "text/event-stream",
2241
+ "Cache-Control": "no-cache",
2242
+ Connection: "keep-alive",
2243
+ "X-Accel-Buffering": "no",
2244
+ });
2245
+ const send = (event, data) => {
2246
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2247
+ };
2248
+ const model = Z_IMAGE_MODEL;
2249
+ try {
2250
+ const mlxgen = await getMlxgenBin();
2251
+ send("progress", {
2252
+ status: "starting",
2253
+ label: `Downloading model ${model}...`,
2254
+ });
2255
+ const proc = spawn([mlxgen, "download", "--model", model], {
2256
+ stdout: "pipe",
2257
+ stderr: "pipe",
2258
+ });
2259
+ activeProc = proc;
2260
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2261
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2262
+ await stdoutPromise;
2263
+ const exitCode = await proc.exited;
2264
+ if (exitCode === 0) {
2265
+ send("complete", { success: true });
2266
+ }
2267
+ else {
2268
+ send("error", {
2269
+ error: stderrText || `Process exited with code ${exitCode}`,
2270
+ exitCode,
2271
+ });
2272
+ }
2273
+ }
2274
+ catch (e) {
2275
+ send("error", { error: String(e) });
2276
+ }
2277
+ finally {
2278
+ activeProc = null;
2279
+ res.end();
2280
+ }
2281
+ });
2282
+ // ========== MLX-Gen: Download FLUX.2 Klein Model ==========
2283
+ app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
2284
+ res.writeHead(200, {
2285
+ "Content-Type": "text/event-stream",
2286
+ "Cache-Control": "no-cache",
2287
+ Connection: "keep-alive",
2288
+ "X-Accel-Buffering": "no",
2289
+ });
2290
+ const send = (event, data) => {
2291
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2292
+ };
2293
+ try {
2294
+ const mlxgen = await getMlxgenBin();
2295
+ send("progress", {
2296
+ status: "starting",
2297
+ label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
2298
+ });
2299
+ const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
2300
+ stdout: "pipe",
2301
+ stderr: "pipe",
2302
+ });
2303
+ activeProc = proc;
2304
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2305
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2306
+ await stdoutPromise;
2307
+ const exitCode = await proc.exited;
2308
+ if (exitCode === 0) {
2309
+ send("complete", { success: true });
2310
+ }
2311
+ else {
2312
+ send("error", {
2313
+ error: stderrText || `Process exited with code ${exitCode}`,
2314
+ exitCode,
2315
+ });
2316
+ }
2317
+ }
2318
+ catch (e) {
2319
+ send("error", { error: String(e) });
2320
+ }
2321
+ finally {
2322
+ activeProc = null;
2323
+ res.end();
2324
+ }
2325
+ });
2326
+ // ========== H3: Download Model ==========
2327
+ app.get("/api/h3/status", (_req, res) => {
2328
+ res.json({ downloaded: isH3ModelDownloaded() });
2329
+ });
2330
+ app.post("/api/h3/download-model", async (_req, res) => {
2331
+ res.writeHead(200, {
2332
+ "Content-Type": "text/event-stream",
2333
+ "Cache-Control": "no-cache",
2334
+ Connection: "keep-alive",
2335
+ "X-Accel-Buffering": "no",
2336
+ });
2337
+ const send = (event, data) => {
2338
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2339
+ };
2340
+ // The `hf download` command must run from inside the mlx-h3 checkout so
2341
+ // its weights land in <mlx-h3>/weights.
2342
+ const featureFolder = join(PYTHON_DIR, "mlx-h3");
2343
+ if (!existsSync(featureFolder)) {
2344
+ mkdirSync(featureFolder, { recursive: true });
2345
+ }
2346
+ try {
2347
+ send("progress", {
2348
+ status: "starting",
2349
+ label: `Downloading model ${H3_MODEL}...`,
2350
+ });
2351
+ const proc = spawn(["hf", "download", H3_MODEL, "--local-dir", "weights"], {
2352
+ cwd: featureFolder,
2353
+ stdout: "pipe",
2354
+ stderr: "pipe",
2355
+ });
2356
+ activeProc = proc;
2357
+ const stdoutPromise = streamToSSE(proc.stdout, "H3 Download", send);
2358
+ const stderrText = await streamToSSE(proc.stderr, "H3 Download", send);
2359
+ await stdoutPromise;
2360
+ const exitCode = await proc.exited;
2361
+ if (exitCode === 0) {
2362
+ send("complete", { success: true });
2363
+ }
2364
+ else {
2365
+ send("error", {
2366
+ error: stderrText || `Process exited with code ${exitCode}`,
2367
+ exitCode,
2368
+ });
2369
+ }
2370
+ }
2371
+ catch (e) {
2372
+ send("error", { error: String(e) });
2373
+ }
2374
+ finally {
2375
+ activeProc = null;
2376
+ res.end();
2377
+ }
2378
+ });
2379
+ // ========== H3: Generate (References-to-Video) ==========
2380
+ app.post("/api/h3/generate", async (req, res) => {
2381
+ const { prompt, refs, projectId, steps = 20, width = 640, height = 448, frames = 121, seed = 42, } = req.body || {};
2382
+ if (!prompt || typeof prompt !== "string" || !prompt.trim()) {
2383
+ res.status(400).json({ error: "Prompt is required" });
2384
+ return;
2385
+ }
2386
+ if (!projectId || !isValidProjectId(String(projectId))) {
2387
+ res.status(400).json({ error: "Invalid project ID" });
2388
+ return;
2389
+ }
2390
+ // Resolve ordered reference media (images, videos, and audio) to safe paths.
2391
+ const mediaRefs = Array.isArray(refs)
2392
+ ? refs.filter((r) => !!r &&
2393
+ (r.kind === "image" || r.kind === "video" || r.kind === "audio") &&
2394
+ typeof r.filename === "string" &&
2395
+ !!r.filename.trim())
2396
+ : [];
2397
+ if (mediaRefs.length === 0) {
2398
+ res.status(400).json({
2399
+ error: "At least one reference image, video, or audio is required. Upload or select one first.",
2400
+ });
2401
+ return;
2402
+ }
2403
+ const resolvedRefs = [];
2404
+ for (const ref of mediaRefs) {
2405
+ const resolved = resolveSafePath(ref.filename.trim(), String(projectId));
2406
+ if (!resolved) {
2407
+ res.status(400).json({
2408
+ error: `Reference ${ref.kind} not found in this project: ${ref.filename}`,
2409
+ });
2410
+ return;
2411
+ }
2412
+ resolvedRefs.push({ kind: ref.kind, path: resolved });
2413
+ }
2414
+ // SSE headers
2415
+ res.writeHead(200, {
2416
+ "Content-Type": "text/event-stream",
2417
+ "Cache-Control": "no-cache",
2418
+ Connection: "keep-alive",
2419
+ "X-Accel-Buffering": "no",
2420
+ });
2421
+ const send = (event, data) => {
2422
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2423
+ };
2424
+ try {
2425
+ const featureFolder = join(PYTHON_DIR, "mlx-h3");
2426
+ if (!existsSync(featureFolder)) {
2427
+ send("error", { error: "mlx-h3 not found. Run setup first." });
2428
+ res.end();
2429
+ return;
2430
+ }
2431
+ const uvPath = await getUvPath();
2432
+ const projectOutputDir = resolveOutputDir(null, String(projectId));
2433
+ if (!projectOutputDir) {
2434
+ send("error", { error: "Invalid output directory." });
2435
+ res.end();
2436
+ return;
2437
+ }
2438
+ const outputFile = `references-${Date.now()}.mp4`;
2439
+ const outputPath = join(projectOutputDir, outputFile);
2440
+ const stepCount = Number(steps) || 20;
2441
+ const videoWidth = Number(width) || 640;
2442
+ const videoHeight = Number(height) || 448;
2443
+ const frameCount = Number(frames) || 121;
2444
+ const seedValue = Number(seed) || 42;
2445
+ send("progress", {
2446
+ status: "starting",
2447
+ label: "Generating references-to-video...",
2448
+ outputFile,
2449
+ settings: {
2450
+ steps: stepCount,
2451
+ width: videoWidth,
2452
+ height: videoHeight,
2453
+ frames: frameCount,
2454
+ seed: seedValue,
2455
+ },
2456
+ });
2457
+ const args = [uvPath, "run", "mlx-h3"];
2458
+ for (const ref of resolvedRefs) {
2459
+ const flag = ref.kind === "image"
2460
+ ? "--ref-image"
2461
+ : ref.kind === "video"
2462
+ ? "--ref-video"
2463
+ : "--ref-audio";
2464
+ args.push(flag, ref.path);
2465
+ }
2466
+ args.push("--steps", String(stepCount), "--width", String(videoWidth), "--height", String(videoHeight), "--frames", String(frameCount), "--seed", String(seedValue), "--output", outputPath, prompt);
2467
+ const proc = spawn(args, {
2468
+ env: process.env,
2469
+ cwd: featureFolder,
2470
+ stdout: "pipe",
2471
+ stderr: "pipe",
2472
+ });
2473
+ activeProc = proc;
2474
+ const stdoutPromise = streamToSSE(proc.stdout, "H3 Generate", send);
2475
+ const stderrText = await streamToSSE(proc.stderr, "H3 Generate", send);
2476
+ await stdoutPromise;
2477
+ const exitCode = await proc.exited;
2478
+ const success = exitCode === 0 && existsSync(outputPath);
2479
+ if (success) {
2480
+ send("complete", {
2481
+ success: true,
2482
+ path: outputPath,
2483
+ filename: outputFile,
2484
+ });
2485
+ }
2486
+ else {
2487
+ send("error", {
2488
+ error: stderrText || `Process exited with code ${exitCode}`,
2489
+ exitCode,
2490
+ });
2491
+ }
2492
+ }
2493
+ catch (e) {
2494
+ send("error", { error: String(e) });
2495
+ }
2496
+ finally {
2497
+ activeProc = null;
2498
+ res.end();
2499
+ }
2500
+ });
2501
+ // ========== MLX-Gen: Generate (Image Edit) ==========
2502
+ app.post("/api/mlxgen/generate", async (req, res) => {
2503
+ const { prompt, imagePath, image, projectId, width, height } = req.body || {};
2504
+ if (!prompt) {
2505
+ res.status(400).json({ error: "Prompt is required" });
2506
+ return;
2507
+ }
2508
+ if (!projectId) {
2509
+ res.status(400).json({ error: "Project ID is required" });
2510
+ return;
2511
+ }
2512
+ // Project IDs are generated by makeId() (alphanumeric + hyphen). Reject
2513
+ // anything else (including `.`, `..`, and path separators) so a malicious
2514
+ // projectId cannot escape OUTPUT_DIR/TEMP_DIR via `../`.
2515
+ if (!/^[a-zA-Z0-9_-]{1,64}$/.test(String(projectId))) {
2516
+ res.status(400).json({ error: "Invalid project ID" });
2517
+ return;
2518
+ }
2519
+ // Resolve the character image. Prefer a client-side preprocessed PNG
2520
+ // (base64), which is written to the temp workspace; otherwise fall back to
2521
+ // a bare filename previously uploaded to this project.
2522
+ let resolvedImage = null;
2523
+ let tempImagePath = null;
2524
+ if (image) {
2525
+ // Decode base64 (strip data URL prefix if present)
2526
+ const base64 = String(image).replace(/^data:image\/\w+;base64,/, "");
2527
+ const buffer = Buffer.from(base64, "base64");
2528
+ const tempDir = join(TEMP_DIR, String(projectId));
2529
+ ensureDir(tempDir);
2530
+ tempImagePath = join(tempDir, `temp-${Date.now()}.png`);
2531
+ writeFileSync(tempImagePath, buffer);
2532
+ resolvedImage = tempImagePath;
2533
+ }
2534
+ else if (imagePath) {
2535
+ resolvedImage = resolveSafePath(imagePath, projectId);
2536
+ }
2537
+ if (!resolvedImage) {
2538
+ res.status(400).json({
2539
+ error: "Image is required. Provide a preprocessed image or a filename previously uploaded to this project.",
2540
+ });
2541
+ return;
2542
+ }
2543
+ res.writeHead(200, {
2544
+ "Content-Type": "text/event-stream",
2545
+ "Cache-Control": "no-cache",
2546
+ Connection: "keep-alive",
2547
+ "X-Accel-Buffering": "no",
2548
+ });
2549
+ const send = (event, data) => {
2550
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2551
+ };
2552
+ try {
2553
+ const mlxgen = await getMlxgenBin();
2554
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
2555
+ ensureDir(projectOutputDir);
2556
+ const outputFile = `mlxgen-${Date.now()}.png`;
2557
+ const outputPath = join(projectOutputDir, outputFile);
2558
+ send("progress", {
2559
+ status: "starting",
2560
+ label: "Generating image...",
2561
+ outputFile,
2562
+ });
2563
+ const args = [
2564
+ mlxgen,
2565
+ "generate",
2566
+ "--model",
2567
+ MLXGEN_MODEL,
2568
+ "--image",
2569
+ resolvedImage,
2570
+ "--prompt",
2571
+ prompt,
2572
+ "--output",
2573
+ outputPath,
2574
+ "--steps",
2575
+ String(40),
2576
+ ];
2577
+ // Optional output size (kept in the same aspect ratio by the client).
2578
+ const outWidth = Number(width);
2579
+ const outHeight = Number(height);
2580
+ if (Number.isInteger(outWidth) &&
2581
+ outWidth > 0 &&
2582
+ Number.isInteger(outHeight) &&
2583
+ outHeight > 0) {
2584
+ args.push("--width", String(outWidth), "--height", String(outHeight));
2585
+ }
2586
+ const proc = spawn(args, {
2587
+ stdout: "pipe",
2588
+ stderr: "pipe",
2589
+ });
2590
+ activeProc = proc;
2591
+ const stdoutPromise = streamToSSE(proc.stdout, "MLXGen", send);
2592
+ const stderrText = await streamToSSE(proc.stderr, "MLXGen", send);
2593
+ await stdoutPromise;
2594
+ const exitCode = await proc.exited;
2595
+ const success = exitCode === 0 && existsSync(outputPath);
2596
+ if (success) {
2597
+ send("complete", {
2598
+ success: true,
2599
+ path: outputPath,
2600
+ filename: outputFile,
2601
+ });
2602
+ }
2603
+ else {
2604
+ send("error", {
2605
+ error: stderrText || `Process exited with code ${exitCode}`,
2606
+ exitCode,
2607
+ });
2608
+ }
2609
+ }
2610
+ catch (e) {
2611
+ send("error", { error: String(e) });
2612
+ }
2613
+ finally {
2614
+ activeProc = null;
2615
+ // Clean up the temporary workspace image now that generation is done.
2616
+ if (tempImagePath) {
2617
+ try {
2618
+ unlinkSync(tempImagePath);
2619
+ }
2620
+ catch {
2621
+ // already removed
2622
+ }
2623
+ try {
2624
+ // Remove the temp dir only when it is empty (no recursive delete).
2625
+ rmSync(join(TEMP_DIR, String(projectId)), { force: true });
2626
+ }
2627
+ catch {
2628
+ // ignore cleanup failures
2629
+ }
2630
+ }
2631
+ res.end();
2632
+ }
2633
+ });
2634
+ // ========== MLX-Gen: Fast Image Edit (FLUX.2 Klein) ==========
2635
+ app.post("/api/mlxgen/fast-image-edit", async (req, res) => {
2636
+ const { prompt, images, projectId } = req.body || {};
2637
+ if (!prompt) {
2638
+ res.status(400).json({ error: "Prompt is required" });
2639
+ return;
2640
+ }
2641
+ if (!projectId) {
2642
+ res.status(400).json({ error: "Project ID is required" });
2643
+ return;
2644
+ }
2645
+ if (!/^[a-zA-Z0-9_-]{1,64}$/.test(String(projectId))) {
2646
+ res.status(400).json({ error: "Invalid project ID" });
2647
+ return;
2648
+ }
2649
+ if (!Array.isArray(images) || images.length === 0) {
2650
+ res
2651
+ .status(400)
2652
+ .json({ error: "At least one reference image is required" });
2653
+ return;
2654
+ }
2655
+ // Decode each base64 reference image into a temp workspace file so the
2656
+ // FLUX model receives them as separate `--image` inputs.
2657
+ const tempDir = join(TEMP_DIR, String(projectId));
2658
+ ensureDir(tempDir);
2659
+ const tempImagePaths = [];
2660
+ try {
2661
+ images.forEach((image, i) => {
2662
+ const base64 = String(image).replace(/^data:image\/\w+;base64,/, "");
2663
+ const buffer = Buffer.from(base64, "base64");
2664
+ const path = join(tempDir, `flux-ref-${Date.now()}-${i}.png`);
2665
+ writeFileSync(path, buffer);
2666
+ tempImagePaths.push(path);
2667
+ });
2668
+ }
2669
+ catch {
2670
+ res.status(400).json({ error: "Invalid reference image data" });
2671
+ return;
2672
+ }
2673
+ res.writeHead(200, {
2674
+ "Content-Type": "text/event-stream",
2675
+ "Cache-Control": "no-cache",
2676
+ Connection: "keep-alive",
2677
+ "X-Accel-Buffering": "no",
2678
+ });
2679
+ const send = (event, data) => {
2680
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2681
+ };
2682
+ try {
2683
+ const mlxgen = await getMlxgenBin();
2684
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
2685
+ ensureDir(projectOutputDir);
2686
+ const outputFile = `flux-edit-${Date.now()}.png`;
2687
+ const outputPath = join(projectOutputDir, outputFile);
2688
+ send("progress", {
2689
+ status: "starting",
2690
+ label: "Generating image...",
2691
+ outputFile,
2692
+ });
2693
+ const args = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
2694
+ console.log(tempImagePaths);
2695
+ for (const path of tempImagePaths) {
2696
+ args.push("--image", path);
2697
+ }
2698
+ args.push("--prompt", prompt, "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps", "5", "--seed", "42", "--width", "1024", "--height", "1024");
2699
+ const proc = spawn(args, {
2700
+ stdout: "pipe",
2701
+ stderr: "pipe",
2702
+ });
2703
+ activeProc = proc;
2704
+ const stdoutPromise = streamToSSE(proc.stdout, "MLXGen", send);
2705
+ const stderrText = await streamToSSE(proc.stderr, "MLXGen", send);
2706
+ await stdoutPromise;
2707
+ const exitCode = await proc.exited;
2708
+ const success = exitCode === 0 && existsSync(outputPath);
2709
+ if (success) {
2710
+ send("complete", {
2711
+ success: true,
2712
+ path: outputPath,
2713
+ filename: outputFile,
2714
+ });
2715
+ }
2716
+ else {
2717
+ send("error", {
2718
+ error: stderrText || `Process exited with code ${exitCode}`,
2719
+ exitCode,
2720
+ });
2721
+ }
2722
+ }
2723
+ catch (e) {
2724
+ send("error", { error: String(e) });
2725
+ }
2726
+ finally {
2727
+ activeProc = null;
2728
+ // Clean up the temporary reference images.
2729
+ for (const path of tempImagePaths) {
2730
+ try {
2731
+ unlinkSync(path);
2732
+ }
2733
+ catch {
2734
+ // already removed
2735
+ }
2736
+ }
2737
+ try {
2738
+ rmSync(tempDir, { force: true });
2739
+ }
2740
+ catch {
2741
+ // ignore cleanup failures
2742
+ }
2743
+ res.end();
2744
+ }
2745
+ });
2746
+ // ========== MLX-Gen: Generate (Text-to-Image) ==========
2747
+ app.post("/api/mlxgen/text-to-image", async (req, res) => {
2748
+ const { prompt, projectId, width, height, steps } = req.body || {};
2749
+ if (!prompt) {
2750
+ res.status(400).json({ error: "Prompt is required" });
2751
+ return;
2752
+ }
2753
+ if (!projectId) {
2754
+ res.status(400).json({ error: "Project ID is required" });
2755
+ return;
2756
+ }
2757
+ if (!/^[a-zA-Z0-9_-]{1,64}$/.test(String(projectId))) {
2758
+ res.status(400).json({ error: "Invalid project ID" });
2759
+ return;
2760
+ }
2761
+ res.writeHead(200, {
2762
+ "Content-Type": "text/event-stream",
2763
+ "Cache-Control": "no-cache",
2764
+ Connection: "keep-alive",
2765
+ "X-Accel-Buffering": "no",
2766
+ });
2767
+ const send = (event, data) => {
2768
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2769
+ };
2770
+ try {
2771
+ const mlxgen = await getMlxgenBin();
2772
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
2773
+ ensureDir(projectOutputDir);
2774
+ const outputFile = `zimage-${Date.now()}.png`;
2775
+ const outputPath = join(projectOutputDir, outputFile);
2776
+ send("progress", {
2777
+ status: "starting",
2778
+ label: "Generating image...",
2779
+ outputFile,
2780
+ });
2781
+ // z-image-turbo is a few-step distillation model; default to 4 steps.
2782
+ const resolvedSteps = Number(steps) > 0 ? Number(steps) : 6;
2783
+ const model = Z_IMAGE_MODEL;
2784
+ const args = [
2785
+ mlxgen,
2786
+ "generate",
2787
+ "--model",
2788
+ model,
2789
+ "--prompt",
2790
+ prompt,
2791
+ "--output",
2792
+ outputPath,
2793
+ "--steps",
2794
+ String(resolvedSteps),
2795
+ ];
2796
+ const outWidth = Number(width);
2797
+ const outHeight = Number(height);
2798
+ if (Number.isInteger(outWidth) &&
2799
+ outWidth > 0 &&
2800
+ Number.isInteger(outHeight) &&
2801
+ outHeight > 0) {
2802
+ args.push("--width", String(outWidth), "--height", String(outHeight));
2803
+ }
2804
+ const proc = spawn(args, {
2805
+ stdout: "pipe",
2806
+ stderr: "pipe",
2807
+ });
2808
+ activeProc = proc;
2809
+ const stdoutPromise = streamToSSE(proc.stdout, "MLXGen", send);
2810
+ const stderrText = await streamToSSE(proc.stderr, "MLXGen", send);
2811
+ await stdoutPromise;
2812
+ const exitCode = await proc.exited;
2813
+ const success = exitCode === 0 && existsSync(outputPath);
2814
+ if (success) {
2815
+ send("complete", {
2816
+ success: true,
2817
+ path: outputPath,
2818
+ filename: outputFile,
2819
+ });
2820
+ }
2821
+ else {
2822
+ send("error", {
2823
+ error: stderrText || `Process exited with code ${exitCode}`,
2824
+ exitCode,
2825
+ });
2826
+ }
2827
+ }
2828
+ catch (e) {
2829
+ send("error", { error: String(e) });
2830
+ }
2831
+ finally {
2832
+ activeProc = null;
2833
+ res.end();
2834
+ }
2835
+ });
2836
+ // ========== Agent (mlx-vlm) ==========
2837
+ app.get("/api/agent/status", (_req, res) => {
2838
+ res.json({
2839
+ installed: isMlxVlmInstalled(),
2840
+ serverRunning: agentServerProc !== null,
2841
+ });
2842
+ });
2843
+ app.post("/api/agent/install", async (_req, res) => {
2844
+ res.writeHead(200, {
2845
+ "Content-Type": "text/event-stream",
2846
+ "Cache-Control": "no-cache",
2847
+ Connection: "keep-alive",
2848
+ "X-Accel-Buffering": "no",
2849
+ });
2850
+ const send = (event, data) => {
2851
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2852
+ };
2853
+ try {
2854
+ const uvPath = await getUvPath();
2855
+ send("progress", {
2856
+ status: "starting",
2857
+ label: "Installing mlx-vlm...",
2858
+ });
2859
+ const proc = spawn([uvPath, "tool", "install", "mlx-vlm"], {
2860
+ stdout: "pipe",
2861
+ stderr: "pipe",
2862
+ });
2863
+ activeProc = proc;
2864
+ const stdoutPromise = streamToSSE(proc.stdout, "Install mlx-vlm", send);
2865
+ const stderrText = await streamToSSE(proc.stderr, "Install mlx-vlm", send);
2866
+ await stdoutPromise;
2867
+ const exitCode = await proc.exited;
2868
+ if (exitCode === 0) {
2869
+ send("complete", { success: true });
2870
+ }
2871
+ else {
2872
+ send("error", {
2873
+ error: stderrText || `Process exited with code ${exitCode}`,
2874
+ exitCode,
2875
+ });
2876
+ }
2877
+ }
2878
+ catch (e) {
2879
+ send("error", { error: String(e) });
2880
+ }
2881
+ finally {
2882
+ activeProc = null;
2883
+ res.end();
2884
+ }
2885
+ });
2886
+ app.post("/api/agent/start", async (req, res) => {
2887
+ const { port, model } = req.body || {};
2888
+ const portNum = Number(port);
2889
+ if (!Number.isInteger(portNum) || portNum < 1 || portNum > 65535) {
2890
+ res
2891
+ .status(400)
2892
+ .json({ error: "Port must be an integer between 1 and 65535" });
2893
+ return;
2894
+ }
2895
+ const modelName = typeof model === "string" && model.trim() ? model.trim() : MLX_VLM_MODEL;
2896
+ res.writeHead(200, {
2897
+ "Content-Type": "text/event-stream",
2898
+ "Cache-Control": "no-cache",
2899
+ Connection: "keep-alive",
2900
+ "X-Accel-Buffering": "no",
2901
+ });
2902
+ const send = (event, data) => {
2903
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2904
+ };
2905
+ try {
2906
+ const bin = await getMlxVlmServerBin();
2907
+ const killedPids = await killProcessOnPort(portNum);
2908
+ if (killedPids.length > 0) {
2909
+ send("log", {
2910
+ text: `Freed port ${portNum} (killed PID(s): ${killedPids.join(", ")})\n`,
2911
+ });
2912
+ }
2913
+ send("progress", {
2914
+ status: "starting",
2915
+ label: `Starting mlx-vlm server (${modelName}) on port ${portNum}...`,
2916
+ });
2917
+ const args = [
2918
+ bin,
2919
+ "--model",
2920
+ modelName,
2921
+ "--port",
2922
+ String(portNum),
2923
+ "--max-tokens",
2924
+ "256000",
2925
+ ];
2926
+ // const draftModel = DRAFT_MODELS[modelName];
2927
+ // if (draftModel) {
2928
+ // args.push(
2929
+ // "--draft-model",
2930
+ // draftModel,
2931
+ // "--draft-kind",
2932
+ // "mtp",
2933
+ // "--draft-block-size",
2934
+ // "4",
2935
+ // );
2936
+ // }
2937
+ const proc = spawn(args, {
2938
+ stdout: "pipe",
2939
+ stderr: "pipe",
2940
+ });
2941
+ agentServerProc = proc;
2942
+ agentServerPort = portNum;
2943
+ const stdoutPromise = streamToSSE(proc.stdout, "Agent", send);
2944
+ const stderrText = await streamToSSE(proc.stderr, "Agent", send);
2945
+ await stdoutPromise;
2946
+ const exitCode = await proc.exited;
2947
+ if (agentStopRequested) {
2948
+ send("complete", { success: true, stopped: true });
2949
+ agentStopRequested = false;
2950
+ }
2951
+ else if (exitCode === 0) {
2952
+ send("complete", { success: true });
2953
+ }
2954
+ else {
2955
+ send("error", {
2956
+ error: stderrText || `Server exited with code ${exitCode}`,
2957
+ exitCode,
2958
+ });
2959
+ }
2960
+ }
2961
+ catch (e) {
2962
+ send("error", { error: String(e) });
2963
+ }
2964
+ finally {
2965
+ agentServerProc = null;
2966
+ agentServerPort = null;
2967
+ res.end();
2968
+ }
2969
+ });
2970
+ app.post("/api/agent/stop", (_req, res) => {
2971
+ agentStopRequested = true;
2972
+ if (agentServerProc) {
2973
+ try {
2974
+ agentServerProc.kill();
2975
+ }
2976
+ catch {
2977
+ // process may already be dead
2978
+ }
2979
+ }
2980
+ agentServerPort = null;
2981
+ res.json({ ok: true });
2982
+ });
2983
+ app.post("/api/agent/open", (req, res) => {
2984
+ const { url } = req.body || {};
2985
+ const target = typeof url === "string" ? url.trim() : "";
2986
+ // Only open localhost http(s) URLs (avoid opening arbitrary schemes/files).
2987
+ if (!/^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?(\/.*)?$/i.test(target)) {
2988
+ res.status(400).json({ error: "Invalid URL" });
2989
+ return;
2990
+ }
2991
+ spawn(["open", target], {
2992
+ stdout: "ignore",
2993
+ stderr: "ignore",
2994
+ onExit: (_proc, exitCode, _signalCode, _error) => {
2995
+ if (exitCode !== 0) {
2996
+ console.error(`open: "open ${target}" exited with code ${exitCode}`);
2997
+ }
2998
+ },
2999
+ });
3000
+ res.json({ ok: true });
3001
+ });
3002
+ // List all projects
3003
+ app.get("/api/open-output", (_req, res) => {
3004
+ try {
3005
+ openInFinder(`${OUTPUT_DIR}`);
3006
+ }
3007
+ catch { }
3008
+ res.json({ ok: true });
3009
+ });
3010
+ // ========== Render: Cancel ==========
3011
+ app.post("/api/render/cancel", (_req, res) => {
3012
+ cancelActiveRender();
3013
+ res.json({ ok: true });
3014
+ });
3015
+ // ========== Project CRUD ==========
3016
+ // List all projects
3017
+ app.get("/api/projects", (_req, res) => {
3018
+ const projects = readProjects();
3019
+ res.json(projects);
3020
+ });
3021
+ // Get single project
3022
+ app.get("/api/projects/:id", (req, res) => {
3023
+ const projects = readProjects();
3024
+ const project = projects.find((p) => p.id === req.params.id);
3025
+ if (!project) {
3026
+ res.status(404).json({ error: "Project not found" });
3027
+ return;
3028
+ }
3029
+ res.json(project);
3030
+ });
3031
+ // Create project
3032
+ app.post("/api/projects", (req, res) => {
3033
+ const { name, description } = req.body || {};
3034
+ if (!name) {
3035
+ res.status(400).json({ error: "Name is required" });
3036
+ return;
3037
+ }
3038
+ const projects = readProjects();
3039
+ const now = new Date().toISOString();
3040
+ const project = {
3041
+ id: makeId(),
3042
+ name: String(name),
3043
+ description: String(description || ""),
3044
+ createdAt: now,
3045
+ updatedAt: now,
3046
+ };
3047
+ // Create project folders
3048
+ ensureDir(join(UPLOAD_DIR, project.id));
3049
+ ensureDir(join(OUTPUT_DIR, project.id));
3050
+ projects.push(project);
3051
+ writeProjects(projects);
3052
+ res.status(201).json(project);
3053
+ });
3054
+ // Update project
3055
+ app.put("/api/projects/:id", (req, res) => {
3056
+ const projects = readProjects();
3057
+ const index = projects.findIndex((p) => p.id === req.params.id);
3058
+ if (index === -1) {
3059
+ res.status(404).json({ error: "Project not found" });
3060
+ return;
3061
+ }
3062
+ const { name, description } = req.body || {};
3063
+ if (name !== undefined)
3064
+ projects[index].name = String(name);
3065
+ if (description !== undefined)
3066
+ projects[index].description = String(description);
3067
+ projects[index].updatedAt = new Date().toISOString();
3068
+ writeProjects(projects);
3069
+ res.json(projects[index]);
3070
+ });
3071
+ // Delete project
3072
+ app.delete("/api/projects/:id", (req, res) => {
3073
+ const projects = readProjects();
3074
+ const index = projects.findIndex((p) => p.id === req.params.id);
3075
+ if (index === -1) {
3076
+ res.status(404).json({ error: "Project not found" });
3077
+ return;
3078
+ }
3079
+ const [removed] = projects.splice(index, 1);
3080
+ writeProjects(projects);
3081
+ res.json(removed);
3082
+ });
3083
+ // ===== Character CRUD =====
3084
+ app.get("/api/characters", (req, res) => {
3085
+ const projectId = String(req.query.projectId ?? "");
3086
+ if (!isValidProjectId(projectId)) {
3087
+ res.status(400).json({ error: "Invalid project ID" });
3088
+ return;
3089
+ }
3090
+ const characters = readCharacters().filter((c) => c.projectId === projectId);
3091
+ res.json(characters);
3092
+ });
3093
+ app.post("/api/characters", (req, res) => {
3094
+ const { projectId, name, filename, source } = req.body || {};
3095
+ if (!projectId || !isValidProjectId(String(projectId))) {
3096
+ res.status(400).json({ error: "Invalid project ID" });
3097
+ return;
3098
+ }
3099
+ if (!filename || !String(filename).trim()) {
3100
+ res.status(400).json({ error: "Filename is required" });
3101
+ return;
3102
+ }
3103
+ const safeFilename = String(filename).split(/[/\\]/).pop() || "";
3104
+ const characters = readCharacters();
3105
+ const character = {
3106
+ id: makeId(),
3107
+ projectId: String(projectId),
3108
+ name: String(name ?? "").trim() || safeFilename,
3109
+ filename: safeFilename,
3110
+ source: source === "generated" ? "generated" : "upload",
3111
+ createdAt: new Date().toISOString(),
3112
+ };
3113
+ characters.push(character);
3114
+ writeCharacters(characters);
3115
+ res.status(201).json(character);
3116
+ });
3117
+ app.put("/api/characters/:id", (req, res) => {
3118
+ const projectId = String(req.query.projectId ?? "");
3119
+ if (!isValidProjectId(projectId)) {
3120
+ res.status(400).json({ error: "Invalid project ID" });
3121
+ return;
3122
+ }
3123
+ const characters = readCharacters();
3124
+ const index = characters.findIndex((c) => c.id === req.params.id && c.projectId === projectId);
3125
+ if (index === -1) {
3126
+ res.status(404).json({ error: "Character not found" });
3127
+ return;
3128
+ }
3129
+ const { name, filename, source } = req.body || {};
3130
+ if (name !== undefined && String(name).trim()) {
3131
+ characters[index].name = String(name).trim();
3132
+ }
3133
+ if (filename !== undefined && String(filename).trim()) {
3134
+ characters[index].filename = String(filename).split(/[/\\]/).pop() || "";
3135
+ characters[index].source =
3136
+ source === "generated" ? "generated" : "upload";
3137
+ }
3138
+ writeCharacters(characters);
3139
+ res.json(characters[index]);
3140
+ });
3141
+ app.delete("/api/characters/:id", (req, res) => {
3142
+ const projectId = String(req.query.projectId ?? "");
3143
+ if (!isValidProjectId(projectId)) {
3144
+ res.status(400).json({ error: "Invalid project ID" });
3145
+ return;
3146
+ }
3147
+ const characters = readCharacters();
3148
+ const index = characters.findIndex((c) => c.id === req.params.id && c.projectId === projectId);
3149
+ if (index === -1) {
3150
+ res.status(404).json({ error: "Character not found" });
3151
+ return;
3152
+ }
3153
+ const [removed] = characters.splice(index, 1);
3154
+ writeCharacters(characters);
3155
+ res.json(removed);
3156
+ });
3157
+ // Save an extracted video frame into the project's extracted-frames folder.
3158
+ app.post("/api/extracted-frames", (req, res) => {
3159
+ const { image, filename, projectId } = req.body || {};
3160
+ if (!image) {
3161
+ res.status(400).json({ error: "Image data is required (base64)" });
3162
+ return;
3163
+ }
3164
+ if (!projectId || !isValidProjectId(String(projectId))) {
3165
+ res.status(400).json({ error: "Invalid project ID" });
3166
+ return;
3167
+ }
3168
+ try {
3169
+ const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
3170
+ const buffer = Buffer.from(base64, "base64");
3171
+ const dir = join(EXTRACTED_FRAMES_DIR, String(projectId));
3172
+ ensureDir(dir);
3173
+ const safeName = (filename || `frame-${Date.now()}.png`).replace(/[^a-zA-Z0-9._-]/g, "_");
3174
+ writeFileSync(join(dir, safeName), buffer);
3175
+ res.json({
3176
+ success: true,
3177
+ path: join(dir, safeName),
3178
+ filename: safeName,
3179
+ size: buffer.length,
3180
+ });
3181
+ }
3182
+ catch (e) {
3183
+ res
3184
+ .status(500)
3185
+ .json({ error: "Failed to save frame", details: String(e) });
3186
+ }
3187
+ });
3188
+ // Save the character sheet: current.png plus a backup copy under backup/.
3189
+ app.post("/api/character-sheet", (req, res) => {
3190
+ const { image, projectId } = req.body || {};
3191
+ if (!image) {
3192
+ res.status(400).json({ error: "Image data is required (base64)" });
3193
+ return;
3194
+ }
3195
+ if (!projectId || !isValidProjectId(String(projectId))) {
3196
+ res.status(400).json({ error: "Invalid project ID" });
3197
+ return;
3198
+ }
3199
+ try {
3200
+ const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
3201
+ const buffer = Buffer.from(base64, "base64");
3202
+ const projectDir = join(CHARACTER_SHEET_DIR, String(projectId));
3203
+ const backupDir = join(projectDir, "backup");
3204
+ ensureDir(backupDir);
3205
+ const backupId = randomUUID();
3206
+ const backupPath = join(backupDir, `${backupId}.png`);
3207
+ const currentPath = join(projectDir, "current.png");
3208
+ writeFileSync(backupPath, buffer);
3209
+ writeFileSync(currentPath, buffer);
3210
+ res.json({
3211
+ success: true,
3212
+ path: currentPath,
3213
+ backupPath,
3214
+ backupFilename: `${backupId}.png`,
3215
+ size: buffer.length,
3216
+ });
3217
+ }
3218
+ catch (e) {
3219
+ res
3220
+ .status(500)
3221
+ .json({ error: "Failed to save character sheet", details: String(e) });
3222
+ }
3223
+ });
3224
+ // List saved character sheet images (current.png) for a project.
3225
+ app.get("/api/projects/:id/character-sheets", (req, res) => {
3226
+ const { id } = req.params;
3227
+ if (!isValidProjectId(id)) {
3228
+ res.status(400).json({ error: "Invalid project ID" });
3229
+ return;
3230
+ }
3231
+ const projectDir = join(CHARACTER_SHEET_DIR, id);
3232
+ const results = [];
3233
+ if (existsSync(projectDir)) {
3234
+ let entries = [];
3235
+ try {
3236
+ entries = readdirSync(projectDir);
3237
+ }
3238
+ catch {
3239
+ entries = [];
3240
+ }
3241
+ for (const entry of entries) {
3242
+ if (!entry.toLowerCase().endsWith(".png"))
3243
+ continue;
3244
+ const fullPath = join(projectDir, entry);
3245
+ try {
3246
+ if (!statSync(fullPath).isFile())
3247
+ continue;
3248
+ }
3249
+ catch {
3250
+ continue;
3251
+ }
3252
+ results.push({
3253
+ filename: entry,
3254
+ url: `/api/files?path=${encodeURIComponent(fullPath)}`,
3255
+ });
3256
+ }
3257
+ }
3258
+ res.json(results);
3259
+ });
3260
+ // Open project folder in Finder
3261
+ app.post("/api/projects/:id/open-folder", (req, res) => {
3262
+ const { type } = req.body || {};
3263
+ const projects = readProjects();
3264
+ const project = projects.find((p) => p.id === req.params.id);
3265
+ if (!project) {
3266
+ res.status(404).json({ error: "Project not found" });
3267
+ return;
3268
+ }
3269
+ let targetPath;
3270
+ switch (type) {
3271
+ case "upload":
3272
+ targetPath = join(UPLOAD_DIR, project.id);
3273
+ break;
3274
+ case "output":
3275
+ targetPath = join(OUTPUT_DIR, project.id);
3276
+ break;
3277
+ default:
3278
+ targetPath = join(OUTPUT_DIR, project.id);
3279
+ }
3280
+ ensureDir(targetPath);
3281
+ try {
3282
+ openInFinder(targetPath);
3283
+ res.json({ success: true, path: targetPath });
3284
+ }
3285
+ catch (e) {
3286
+ res
3287
+ .status(500)
3288
+ .json({ error: "Failed to open folder", details: String(e) });
3289
+ }
3290
+ });
3291
+ // Open project in Finder (root project folder - opens output dir)
3292
+ app.post("/api/projects/:id/open-in-finder", (_req, res) => {
3293
+ const projects = readProjects();
3294
+ const project = projects.find((p) => p.id === _req.params.id);
3295
+ if (!project) {
3296
+ res.status(404).json({ error: "Project not found" });
3297
+ return;
3298
+ }
3299
+ const projectOutputDir = join(OUTPUT_DIR, project.id);
3300
+ ensureDir(projectOutputDir);
3301
+ try {
3302
+ openInFinder(projectOutputDir);
3303
+ res.json({ success: true, path: projectOutputDir });
3304
+ }
3305
+ catch (e) {
3306
+ res
3307
+ .status(500)
3308
+ .json({ error: "Failed to open in Finder", details: String(e) });
3309
+ }
3310
+ });
3311
+ }