@effectnode/media 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/backend/movie-backend/agent/prompt/script.md +213 -0
  2. package/dist/backend/movie-backend/core.js +33 -0
  3. package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
  4. package/dist/backend/movie-backend/generation-queue.js +82 -6
  5. package/dist/backend/movie-backend/render-media.d.ts +89 -1
  6. package/dist/backend/movie-backend/render-media.js +772 -476
  7. package/frontend/src/movie-app/components/EditorTabs/AdvancedVoiceCloneTab.tsx +366 -0
  8. package/frontend/src/movie-app/components/EditorTabs/AudioToVideoTab.tsx +406 -0
  9. package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +67 -51
  10. package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +155 -1
  11. package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +33 -13
  12. package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +26 -5
  13. package/frontend/src/movie-app/components/EditorTabs/UpscaleTab.tsx +342 -0
  14. package/frontend/src/movie-app/components/EditorTabs/VoiceCloneTab.tsx +365 -0
  15. package/frontend/src/movie-app/components/ProjectEditorPage.tsx +131 -126
  16. package/frontend/src/movie-app/stores/advancedVoiceCloneStore.ts +207 -0
  17. package/frontend/src/movie-app/stores/aiModelStore.ts +25 -2
  18. package/frontend/src/movie-app/stores/audioToVideoStore.ts +274 -0
  19. package/frontend/src/movie-app/stores/generationStore.ts +250 -345
  20. package/frontend/src/movie-app/stores/movieStudioStore.ts +10 -3
  21. package/frontend/src/movie-app/stores/projectStore.ts +7 -3
  22. package/frontend/src/movie-app/stores/queueStore.ts +48 -1
  23. package/frontend/src/movie-app/stores/upscaleStore.ts +118 -0
  24. package/frontend/src/movie-app/stores/voiceCloneStore.ts +227 -0
  25. package/package.json +1 -1
  26. package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +0 -913
  27. package/frontend/src/movie-app/components/EditorTabs/CharacterSheet.tsx +0 -234
  28. package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +0 -305
  29. package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +0 -249
  30. package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +0 -267
  31. package/frontend/src/movie-app/lib/batchVoiceStorage.ts +0 -75
  32. package/frontend/src/movie-app/stores/batchVoiceStore.ts +0 -990
  33. package/frontend/src/movie-app/stores/sceneVisualStore.ts +0 -251
@@ -1,7 +1,6 @@
1
1
  import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync, readdirSync, statSync, unlinkSync, rmSync, copyFileSync, } from "node:fs";
2
- import { randomUUID } from "node:crypto";
3
2
  import { homedir } from "node:os";
4
- import { join, sep } from "node:path";
3
+ import { dirname, join, sep } from "node:path";
5
4
  import { spawn, whichSync, mimeType } from "./process.js";
6
5
  // Track the currently active spawn process so it can be cancelled
7
6
  let activeProc = null;
@@ -17,8 +16,6 @@ const APP_DATA_DIR = join(homedir(), "media-studio");
17
16
  const OUTPUT_DIR = join(APP_DATA_DIR, "output");
18
17
  const UPLOAD_DIR = join(APP_DATA_DIR, "upload");
19
18
  const AGENT_UPLOAD_DIR = join(APP_DATA_DIR, "agent-upload");
20
- const EXTRACTED_FRAMES_DIR = join(APP_DATA_DIR, "extracted-frames");
21
- const CHARACTER_SHEET_DIR = join(APP_DATA_DIR, "character-sheet");
22
19
  const AGENTS_DIR = join(APP_DATA_DIR, "agents");
23
20
  const JSON_DIR = join(APP_DATA_DIR, "json");
24
21
  const PYTHON_DIR = join(APP_DATA_DIR, "python-src");
@@ -27,7 +24,10 @@ const PROJECTS_FILE = join(JSON_DIR, "projects.json");
27
24
  const CHARACTERS_FILE = join(JSON_DIR, "characters.json");
28
25
  const Z_IMAGE_MODEL = "AbstractFramework/z-image-turbo-8bit";
29
26
  const FLUX_KLEIN_MODEL = "AbstractFramework/flux.2-klein-4b-8bit";
27
+ const SEEDVR2_MODEL = "AbstractFramework/seedvr2-7b-8bit";
30
28
  const MLX_VLM_MODEL = "mlx-community/gemma-4-e4b-it-8bit";
29
+ const LTX_MODEL_HIGH_QUALITY = "dgrauet/ltx-2.3-mlx";
30
+ const LTX_MODEL_STANDARD = "dgrauet/ltx-2.3-mlx-q8";
31
31
  const VIDEO_STAGE_FLAGS = {
32
32
  distilled: "--distilled",
33
33
  "one-stage": "--one-stage",
@@ -43,6 +43,12 @@ function stageFlagFor(mode) {
43
43
  ? (VIDEO_STAGE_FLAGS[mode] ?? "--distilled")
44
44
  : "--distilled";
45
45
  }
46
+ /** Resolve a ltx-2-mlx model id, falling back to the q8 standard model. */
47
+ function resolveLtxModel(model) {
48
+ return String(model) === LTX_MODEL_HIGH_QUALITY
49
+ ? LTX_MODEL_HIGH_QUALITY
50
+ : LTX_MODEL_STANDARD;
51
+ }
46
52
  // ========== Project Helpers ==========
47
53
  function ensureDir(dir) {
48
54
  if (!existsSync(dir)) {
@@ -141,12 +147,11 @@ let _allowedRealDirs = null;
141
147
  function getAllowedRealDirs() {
142
148
  if (_allowedRealDirs)
143
149
  return _allowedRealDirs;
144
- [OUTPUT_DIR, UPLOAD_DIR, AGENT_UPLOAD_DIR, CHARACTER_SHEET_DIR].forEach((d) => ensureDir(d));
150
+ [OUTPUT_DIR, UPLOAD_DIR, AGENT_UPLOAD_DIR].forEach((d) => ensureDir(d));
145
151
  _allowedRealDirs = [
146
152
  realpathSync(OUTPUT_DIR) + sep,
147
153
  realpathSync(UPLOAD_DIR) + sep,
148
154
  realpathSync(AGENT_UPLOAD_DIR) + sep,
149
- realpathSync(CHARACTER_SHEET_DIR) + sep,
150
155
  ];
151
156
  return _allowedRealDirs;
152
157
  }
@@ -190,22 +195,6 @@ function resolveSafePath(candidate, projectId) {
190
195
  }
191
196
  return null;
192
197
  }
193
- /** Resolve and validate a user-supplied video filename. Only bare .mp4 names in the output dir. */
194
- function resolveSafeVideoPath(candidate, projectId) {
195
- const base = candidate.split(/[/\\]/).pop() || candidate;
196
- if (base !== candidate || base.includes("..") || base.startsWith(".")) {
197
- return null;
198
- }
199
- if (!base.toLowerCase().endsWith(".mp4"))
200
- return null;
201
- const candidatePath = join(OUTPUT_DIR, projectId, base);
202
- if (!existsSync(candidatePath))
203
- return null;
204
- const resolved = realpathSync(candidatePath);
205
- if (isPathAllowed(resolved))
206
- return resolved;
207
- return null;
208
- }
209
198
  /**
210
199
  * Resolve the `mlxgen` executable installed via `uv tool install --upgrade mlx-gen`.
211
200
  * uv tool installs binaries into `~/.local/bin`; fall back to relying on PATH.
@@ -222,6 +211,63 @@ async function getMlxgenBin() {
222
211
  }
223
212
  return "mlxgen";
224
213
  }
214
+ /** Resolve the `dots-tts` executable installed via `uv tool install dots-tts`. */
215
+ async function getDotsTtsBin() {
216
+ const candidates = [
217
+ join(homedir(), ".local", "bin", "dots-tts"),
218
+ "/opt/homebrew/bin/dots-tts",
219
+ "/usr/local/bin/dots-tts",
220
+ ];
221
+ for (const p of candidates) {
222
+ if (existsSync(p))
223
+ return p;
224
+ }
225
+ return "dots-tts";
226
+ }
227
+ /** Resolve the `mlx_whisper` executable installed via `uv tool install mlx-whisper`. */
228
+ async function getMlxWhisperBin() {
229
+ const candidates = [
230
+ join(homedir(), ".local", "bin", "mlx_whisper"),
231
+ "/opt/homebrew/bin/mlx_whisper",
232
+ "/usr/local/bin/mlx_whisper",
233
+ ];
234
+ for (const p of candidates) {
235
+ if (existsSync(p))
236
+ return p;
237
+ }
238
+ return "mlx_whisper";
239
+ }
240
+ /** The cloned dots-tts-mlx project directory. */
241
+ const DOTS_TTS_FOLDER = join(APP_DATA_DIR, "python-src", "dots-tts-mlx");
242
+ /** Base directory where dots-tts MLX weights live (inside the project folder). */
243
+ const DOTS_TTS_WEIGHTS_DIR = join(DOTS_TTS_FOLDER, "dots-tts-mlx-weights");
244
+ /**
245
+ * Resolve a dots-tts `--model` value. Accepts `./dots-tts-mlx-weights/mf-int4`
246
+ * (relative to the dots-tts-mlx folder), a bare variant (`mf-int4`), or an
247
+ * explicit path.
248
+ */
249
+ function resolveDotsTtsModel(model) {
250
+ if (!model)
251
+ return join(DOTS_TTS_WEIGHTS_DIR, "mf-int4");
252
+ if (model.startsWith("./"))
253
+ return join(DOTS_TTS_FOLDER, model.slice(2));
254
+ if (model.includes("/"))
255
+ return model;
256
+ return join(DOTS_TTS_WEIGHTS_DIR, model);
257
+ }
258
+ /** True when the dots-tts `mf-int4` weights have been downloaded. */
259
+ function isDotsTtsModelDownloaded() {
260
+ return existsSync(join(DOTS_TTS_WEIGHTS_DIR, "mf-int4"));
261
+ }
262
+ /** Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel). */
263
+ async function getFfmpegBin() {
264
+ const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
265
+ for (const p of candidates) {
266
+ if (existsSync(p))
267
+ return p;
268
+ }
269
+ return "ffmpeg";
270
+ }
225
271
  /** True when the `mlxgen` executable is installed (known paths or PATH). */
226
272
  function isMlxgenInstalled() {
227
273
  const candidates = [
@@ -296,18 +342,6 @@ function isMlxVlmInstalled() {
296
342
  return false;
297
343
  }
298
344
  }
299
- /**
300
- * Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel).
301
- * Falls back to relying on PATH when neither known location exists.
302
- */
303
- async function getFfmpegBin() {
304
- const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
305
- for (const p of candidates) {
306
- if (existsSync(p))
307
- return p;
308
- }
309
- return "ffmpeg";
310
- }
311
345
  /** Fixed filename mlx_audio.tts.generate writes its output clip as. */
312
346
  const TTS_OUTPUT_FILENAME = "audio_000.mp3";
313
347
  /**
@@ -577,6 +611,76 @@ export async function generateSceneVideo(uvPath, projectId, scene, characters, o
577
611
  url: `/api/files?path=${encodeURIComponent(videoPath)}`,
578
612
  };
579
613
  }
614
+ /**
615
+ * Generate a video from a project image via LTX-2.3. `imagePath` must be a bare
616
+ * filename previously uploaded/generated for this project. Used by the queue
617
+ * worker for the "Scene Video Generation" tab.
618
+ */
619
+ export async function generateImageToVideo(uvPath, projectId, params, onLog) {
620
+ const { prompt, imagePath, mode } = params;
621
+ if (!isValidProjectId(projectId))
622
+ return { error: "Invalid project ID" };
623
+ if (!prompt || !prompt.trim())
624
+ return { error: "Prompt is required" };
625
+ if (!imagePath)
626
+ return { error: "Image path is required" };
627
+ const resolvedImage = resolveSafePath(imagePath, projectId);
628
+ if (!resolvedImage) {
629
+ return {
630
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
631
+ };
632
+ }
633
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
634
+ if (!existsSync(ltxFolder)) {
635
+ return { error: "ltx-2-mlx not found. Run setup first." };
636
+ }
637
+ const projectOutputDir = resolveOutputDir(undefined, projectId);
638
+ if (!projectOutputDir)
639
+ return { error: "Invalid output directory." };
640
+ const outputFile = `video-${Date.now()}.mp4`;
641
+ const outputPath = join(projectOutputDir, outputFile);
642
+ const videoWidth = Number(params.width) || 480;
643
+ const videoHeight = Number(params.height) || 480;
644
+ const videoFrames = Number(params.frames) || 121;
645
+ const videoFps = Number(params.frameRate) || 24;
646
+ const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 30);
647
+ const stage2Steps = Math.max(1, Math.round(Number(params.stage2Steps)) || 3);
648
+ const result = await runCommand([
649
+ uvPath,
650
+ "run",
651
+ "ltx-2-mlx",
652
+ "generate",
653
+ "--model",
654
+ resolveLtxModel(params.model),
655
+ "--prompt",
656
+ prompt.trim(),
657
+ stageFlagFor(mode),
658
+ "--stage1-steps",
659
+ String(stage1Steps),
660
+ "--stage2-steps",
661
+ String(stage2Steps),
662
+ "--frames",
663
+ String(videoFrames),
664
+ "--width",
665
+ String(videoWidth),
666
+ "--height",
667
+ String(videoHeight),
668
+ "--frame-rate",
669
+ String(videoFps),
670
+ "--image",
671
+ resolvedImage,
672
+ "--output",
673
+ outputPath,
674
+ ], { cwd: ltxFolder, onLog });
675
+ if (!result.success || !existsSync(outputPath)) {
676
+ return { error: result.output || "Video generation failed" };
677
+ }
678
+ backupFile(outputPath, projectId);
679
+ return {
680
+ filename: outputFile,
681
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
682
+ };
683
+ }
580
684
  /** Normalize a value into a safe filesystem slug. */
581
685
  function slugify(v) {
582
686
  return String(v || "")
@@ -587,10 +691,11 @@ function slugify(v) {
587
691
  * Generate a single scene image via fast-image-edit (FLUX.2 Klein), using the
588
692
  * already-generated character/place images referenced by the scene's slugs.
589
693
  */
590
- export async function generateSceneImage(projectId, scene, onLog) {
694
+ export async function generateSceneImage(projectId, scene, steps, onLog) {
591
695
  const s = slugify(scene?.slug);
592
696
  if (!s)
593
697
  return { error: "Invalid scene slug" };
698
+ const stepCount = Math.max(1, Number(steps) || 6);
594
699
  const outputDir = join(OUTPUT_DIR, projectId);
595
700
  const mlxgen = await getMlxgenBin();
596
701
  const refImages = [];
@@ -614,7 +719,7 @@ export async function generateSceneImage(projectId, scene, onLog) {
614
719
  const fluxArgs = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
615
720
  for (const p of refImages)
616
721
  fluxArgs.push("--image", p);
617
- fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", "5", "--seed", "42", "--width", "1024", "--height", "1024");
722
+ fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
618
723
  const result = await runCommand(fluxArgs, { onLog });
619
724
  if (!result.success || !existsSync(sceneImagePath)) {
620
725
  return { error: result.output || `Failed to generate scene image ${s}` };
@@ -625,6 +730,439 @@ export async function generateSceneImage(projectId, scene, onLog) {
625
730
  url: `/api/files?path=${encodeURIComponent(sceneImagePath)}`,
626
731
  };
627
732
  }
733
+ /**
734
+ * Upscale/refine a project image via mlxgen (SeedVR2). `imagePath` must be a
735
+ * bare filename previously uploaded/generated for this project. `resolution` is
736
+ * either "1x" (refine at native resolution) or "2048" (upscale to 2048px).
737
+ */
738
+ export async function generateUpscale(projectId, imagePath, resolution, onLog) {
739
+ if (!isValidProjectId(projectId))
740
+ return { error: "Invalid project ID" };
741
+ if (!imagePath)
742
+ return { error: "Image path is required" };
743
+ const resolvedImage = resolveSafePath(imagePath, projectId);
744
+ if (!resolvedImage) {
745
+ return {
746
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
747
+ };
748
+ }
749
+ const target = /^(1x|\d+)$/.test(resolution) ? resolution : "1x";
750
+ const mlxgen = await getMlxgenBin();
751
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
752
+ ensureDir(projectOutputDir);
753
+ const outputFile = `upscale-${target}-${Date.now()}.png`;
754
+ const outputPath = join(projectOutputDir, outputFile);
755
+ const result = await runCommand([
756
+ mlxgen,
757
+ "upscale",
758
+ "--model",
759
+ SEEDVR2_MODEL,
760
+ "--image-path",
761
+ resolvedImage,
762
+ "--resolution",
763
+ target,
764
+ "--seed",
765
+ "42",
766
+ "--mlx-cache-limit-gb",
767
+ "100",
768
+ "--output",
769
+ outputPath,
770
+ ], { onLog });
771
+ if (!result.success || !existsSync(outputPath)) {
772
+ return { error: result.output || "Upscale failed" };
773
+ }
774
+ backupFile(outputPath, projectId);
775
+ return {
776
+ filename: outputFile,
777
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
778
+ };
779
+ }
780
+ /**
781
+ * Clone a reference voice and speak `text` via mlx_audio.tts.generate. `refAudioPath`
782
+ * must be a bare filename previously uploaded to this project. `quality` is "low"
783
+ * or "high" (maps to a TTS model). Output is saved under <output>/voices/.
784
+ */
785
+ export async function generateVoiceClone(uvPath, projectId, text, refAudioPath, quality, onLog) {
786
+ if (!isValidProjectId(projectId))
787
+ return { error: "Invalid project ID" };
788
+ if (!refAudioPath)
789
+ return { error: "Reference audio is required" };
790
+ // Sanitize the transcript: strip control characters (incl. newlines) and
791
+ // collapse whitespace. spawn() runs with shell:false (array args), so shell
792
+ // metacharacters cannot execute, but this keeps `--text` a single well-formed
793
+ // argument and out of the terminal log.
794
+ const cleanText = text
795
+ .replace(/[\u0000-\u001f\u007f]/g, " ")
796
+ .replace(/\s+/g, " ")
797
+ .trim();
798
+ if (!cleanText)
799
+ return { error: "Text is required" };
800
+ const resolvedRef = resolveSafePath(refAudioPath, projectId);
801
+ if (!resolvedRef) {
802
+ return {
803
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
804
+ };
805
+ }
806
+ const model = quality === "low" ? TTS_MODELS.low : TTS_MODELS.high;
807
+ const projectOutputDir = resolveOutputDir(undefined, projectId);
808
+ if (!projectOutputDir)
809
+ return { error: "Invalid output directory." };
810
+ const voiceId = `voice-${Date.now()}`;
811
+ const voiceDir = join(projectOutputDir, "voices", voiceId);
812
+ ensureDir(voiceDir);
813
+ const result = await runCommand([
814
+ uvPath,
815
+ "run",
816
+ "mlx_audio.tts.generate",
817
+ "--model",
818
+ model,
819
+ "--text",
820
+ cleanText,
821
+ "--ref_audio",
822
+ resolvedRef,
823
+ "--output",
824
+ voiceDir,
825
+ "--audio_format",
826
+ "mp3",
827
+ "--play",
828
+ "--instruct",
829
+ "slow down speech",
830
+ ], { cwd: voiceDir, onLog });
831
+ if (!result.success) {
832
+ return { error: result.output || "Voice generation failed" };
833
+ }
834
+ const path = resolveAudioFile(voiceDir);
835
+ if (!path) {
836
+ return { error: "TTS completed but no audio file was produced" };
837
+ }
838
+ const filename = path.split(sep).pop() || TTS_OUTPUT_FILENAME;
839
+ // Persist a per-voice metadata file so the generated-voice list can be rebuilt
840
+ // from the folder structure (each voice in its own folder with meta.json), with
841
+ // no central JSON index.
842
+ const meta = {
843
+ id: voiceId,
844
+ transcript: cleanText,
845
+ quality,
846
+ refAudioFilename: refAudioPath,
847
+ filename,
848
+ createdAt: new Date().toISOString(),
849
+ };
850
+ writeFileSync(join(voiceDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
851
+ return {
852
+ filename,
853
+ url: `/api/files?path=${encodeURIComponent(path)}`,
854
+ };
855
+ }
856
+ /** Find the newest video file (mp4/webm/mov) inside a directory. */
857
+ function findNewestVideo(dir) {
858
+ let entries = [];
859
+ try {
860
+ entries = readdirSync(dir);
861
+ }
862
+ catch {
863
+ return null;
864
+ }
865
+ let newest = null;
866
+ for (const name of entries) {
867
+ const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
868
+ if (ext !== ".mp4" && ext !== ".webm" && ext !== ".mov")
869
+ continue;
870
+ const full = join(dir, name);
871
+ try {
872
+ const st = statSync(full);
873
+ if (!st.isFile())
874
+ continue;
875
+ if (!newest || st.mtimeMs > newest.mtime) {
876
+ newest = { path: full, mtime: st.mtimeMs };
877
+ }
878
+ }
879
+ catch {
880
+ // skip unreadable entries
881
+ }
882
+ }
883
+ return newest ? newest.path : null;
884
+ }
885
+ /**
886
+ * Generate a video from an image + audio via ltx-2-mlx `a2v` (audio-to-video).
887
+ * `imagePath` and `audioPath` must be bare filenames previously uploaded/generated
888
+ * for this project. `stage1Steps` maps to `--stage1-steps` (15 default, 30 HD).
889
+ */
890
+ export async function generateAudioToVideo(uvPath, projectId, params, onLog) {
891
+ if (!isValidProjectId(projectId))
892
+ return { error: "Invalid project ID" };
893
+ if (!params.imagePath)
894
+ return { error: "Image path is required" };
895
+ if (!params.audioPath)
896
+ return { error: "Audio path is required" };
897
+ const resolvedImage = resolveSafePath(params.imagePath, projectId);
898
+ if (!resolvedImage) {
899
+ return {
900
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
901
+ };
902
+ }
903
+ const resolvedAudio = resolveSafePath(params.audioPath, projectId);
904
+ if (!resolvedAudio) {
905
+ return {
906
+ error: "Invalid audio path. Provide a filename previously uploaded to this project.",
907
+ };
908
+ }
909
+ // Collapse whitespace so the prompt stays a single well-formed CLI argument.
910
+ const cleanPrompt = String(params.prompt || "")
911
+ .replace(/\s+/g, " ")
912
+ .trim() || "scene";
913
+ const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 15);
914
+ // 1 second = 24 frames, plus a terminal frame (24n + 1).
915
+ const frames = Math.max(1, Math.round(Number(params.frames)) || 25);
916
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
917
+ if (!existsSync(ltxFolder)) {
918
+ return { error: "ltx-2-mlx not found. Run setup first." };
919
+ }
920
+ const outputDir = join(OUTPUT_DIR, projectId, `a2v-${Date.now()}`);
921
+ ensureDir(outputDir);
922
+ const result = await runCommand([
923
+ uvPath,
924
+ "run",
925
+ "ltx-2-mlx",
926
+ "a2v",
927
+ "--image",
928
+ resolvedImage,
929
+ "--audio",
930
+ resolvedAudio,
931
+ "--frame-rate",
932
+ "24",
933
+ "--frames",
934
+ String(frames),
935
+ "--output",
936
+ outputDir,
937
+ "--prompt",
938
+ cleanPrompt,
939
+ "--stage1-steps",
940
+ String(stage1Steps),
941
+ "--stage2-steps",
942
+ "3",
943
+ ], { cwd: ltxFolder, onLog });
944
+ if (!result.success) {
945
+ return { error: result.output || "Audio-to-video generation failed" };
946
+ }
947
+ const videoPath = findNewestVideo(outputDir);
948
+ if (!videoPath) {
949
+ return { error: "a2v completed but no video file was produced" };
950
+ }
951
+ backupFile(videoPath, projectId);
952
+ return {
953
+ filename: videoPath.split(sep).pop() || "a2v.mp4",
954
+ url: `/api/files?path=${encodeURIComponent(videoPath)}`,
955
+ };
956
+ }
957
+ /**
958
+ * Clone a reference voice via dots-tts. Output is written to
959
+ * <output>/<projectId>/dots-tts/ as `<prefix>_000.wav`.
960
+ */
961
+ export async function generateAdvancedVoiceClone(projectId, params, onLog) {
962
+ if (!isValidProjectId(projectId))
963
+ return { error: "Invalid project ID" };
964
+ const cleanText = String(params.text || "")
965
+ .replace(/\s+/g, " ")
966
+ .trim();
967
+ if (!cleanText)
968
+ return { error: "Text is required" };
969
+ if (!params.refAudioPath)
970
+ return { error: "Reference audio is required" };
971
+ const resolvedRef = resolveSafePath(params.refAudioPath, projectId);
972
+ if (!resolvedRef) {
973
+ return {
974
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
975
+ };
976
+ }
977
+ const language = String(params.language || "YUE").trim() || "YUE";
978
+ const prefix = String(params.outPrefix || "voice")
979
+ .replace(/[^a-zA-Z0-9_-]/g, "_")
980
+ .slice(0, 64) || "voice";
981
+ const model = resolveDotsTtsModel(String(params.model || ""));
982
+ // dots-tts and mlx_whisper both expect a WAV reference — convert mp3/other
983
+ // formats to 16-bit PCM WAV first.
984
+ let refAudio = resolvedRef;
985
+ if (!resolvedRef.toLowerCase().endsWith(".wav")) {
986
+ const tempDir = join(TEMP_DIR, String(projectId));
987
+ ensureDir(tempDir);
988
+ const convertedRef = join(tempDir, `avc-ref-${Date.now()}.wav`);
989
+ const ffmpegBin = await getFfmpegBin();
990
+ const conv = await runCommand([
991
+ ffmpegBin,
992
+ "-y",
993
+ "-i",
994
+ resolvedRef,
995
+ "-c:a",
996
+ "pcm_s16le",
997
+ "-ar",
998
+ "44100",
999
+ "-ac",
1000
+ "2",
1001
+ convertedRef,
1002
+ ], { onLog });
1003
+ if (!conv.success || !existsSync(convertedRef)) {
1004
+ return { error: "Failed to convert reference audio to WAV" };
1005
+ }
1006
+ refAudio = convertedRef;
1007
+ }
1008
+ // dots-tts requires `--ref-text` (the reference audio's transcript). Transcribe
1009
+ // the WAV reference with mlx_whisper to supply it.
1010
+ const whisperBin = await getMlxWhisperBin();
1011
+ const transcribeDir = join(TEMP_DIR, String(projectId));
1012
+ ensureDir(transcribeDir);
1013
+ const whisperResult = await runCommand([whisperBin, refAudio, "--output-dir", transcribeDir], { onLog });
1014
+ const refTxtPath = join(transcribeDir, (refAudio.split(sep).pop() || "ref").replace(/\.[^.]+$/, "") + ".txt");
1015
+ const refText = whisperResult.success && existsSync(refTxtPath)
1016
+ ? readFileSync(refTxtPath, "utf-8").replace(/\s+/g, " ").trim()
1017
+ : "";
1018
+ if (!refText) {
1019
+ return { error: "Failed to transcribe reference audio (mlx_whisper)" };
1020
+ }
1021
+ const dotsTtsBin = await getDotsTtsBin();
1022
+ // Mirror the voice-clone tab's timestamped-folder layout so outputs never
1023
+ // overwrite each other.
1024
+ const voiceId = `voice-${Date.now()}`;
1025
+ const outDir = join(OUTPUT_DIR, projectId, "dots-tts", voiceId);
1026
+ ensureDir(outDir);
1027
+ console.log("debug-info:", [
1028
+ dotsTtsBin,
1029
+ "--model",
1030
+ model,
1031
+ "--text",
1032
+ cleanText || "please provide text",
1033
+ "--ref-audio",
1034
+ refAudio,
1035
+ "--ref-text",
1036
+ refText,
1037
+ "--language",
1038
+ language,
1039
+ "--out-path",
1040
+ outDir,
1041
+ "--out-prefix",
1042
+ prefix,
1043
+ ]);
1044
+ const result = await runCommand([
1045
+ dotsTtsBin,
1046
+ "--model",
1047
+ model,
1048
+ "--text",
1049
+ cleanText || "please provide text",
1050
+ "--ref-audio",
1051
+ refAudio,
1052
+ "--ref-text",
1053
+ refText,
1054
+ "--language",
1055
+ language,
1056
+ "--out-path",
1057
+ outDir,
1058
+ "--out-prefix",
1059
+ prefix,
1060
+ ], { cwd: dirname(dotsTtsBin), onLog });
1061
+ if (!result.success) {
1062
+ return { error: result.output || "Advanced voice clone failed" };
1063
+ }
1064
+ const outputFile = `${prefix}_000.wav`;
1065
+ const outputPath = join(outDir, outputFile);
1066
+ if (!existsSync(outputPath)) {
1067
+ return { error: `Expected output ${outputFile} was not produced` };
1068
+ }
1069
+ // Mirror the voice-clone tab's on-disk conventions: a timestamped backup copy
1070
+ // plus a per-voice meta.json so the output can be listed without a central index.
1071
+ backupFile(outputPath, projectId);
1072
+ const meta = {
1073
+ id: voiceId,
1074
+ transcript: cleanText,
1075
+ refText,
1076
+ language,
1077
+ model,
1078
+ refAudioFilename: params.refAudioPath,
1079
+ outPrefix: prefix,
1080
+ filename: outputFile,
1081
+ createdAt: new Date().toISOString(),
1082
+ };
1083
+ writeFileSync(join(outDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
1084
+ return {
1085
+ filename: outputFile,
1086
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
1087
+ };
1088
+ }
1089
+ /**
1090
+ * Generate a composite image via fast-image-edit (FLUX.2 Klein). `images` are
1091
+ * base64 data URLs that are decoded into temp files and passed to the model as
1092
+ * separate `--image` inputs. Used by the generation queue worker.
1093
+ */
1094
+ export async function generateFastImageEditImage(projectId, prompt, images, steps, upscaleResolution, onLog) {
1095
+ if (!isValidProjectId(projectId))
1096
+ return { error: "Invalid project ID" };
1097
+ if (!prompt || !prompt.trim())
1098
+ return { error: "Prompt is required" };
1099
+ if (!Array.isArray(images) || images.length === 0) {
1100
+ return { error: "At least one reference image is required" };
1101
+ }
1102
+ const stepCount = Math.max(1, Number(steps) || 4);
1103
+ // Decode each base64 reference image into a temp workspace file so the
1104
+ // FLUX model receives them as separate `--image` inputs.
1105
+ const tempDir = join(TEMP_DIR, String(projectId));
1106
+ ensureDir(tempDir);
1107
+ const tempImagePaths = [];
1108
+ try {
1109
+ images.forEach((image, i) => {
1110
+ const base64 = String(image).replace(/^data:image\/\w+;base64,/, "");
1111
+ const buffer = Buffer.from(base64, "base64");
1112
+ const path = join(tempDir, `flux-ref-${Date.now()}-${i}.png`);
1113
+ writeFileSync(path, buffer);
1114
+ tempImagePaths.push(path);
1115
+ });
1116
+ }
1117
+ catch {
1118
+ return { error: "Invalid reference image data" };
1119
+ }
1120
+ try {
1121
+ const mlxgen = await getMlxgenBin();
1122
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
1123
+ ensureDir(projectOutputDir);
1124
+ const outputFile = `flux-edit-${Date.now()}.png`;
1125
+ const outputPath = join(projectOutputDir, outputFile);
1126
+ const args = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
1127
+ for (const path of tempImagePaths)
1128
+ args.push("--image", path);
1129
+ args.push("--prompt", prompt.trim(), "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
1130
+ const result = await runCommand(args, { onLog });
1131
+ if (!result.success || !existsSync(outputPath)) {
1132
+ return { error: result.output || "Fast image edit failed" };
1133
+ }
1134
+ backupFile(outputPath, projectId);
1135
+ // Optionally upscale the generated result (1x / 1500px / 2000px).
1136
+ if (upscaleResolution && upscaleResolution !== "none") {
1137
+ if (onLog)
1138
+ onLog(`Upscaling result (${upscaleResolution})…\n`);
1139
+ const upscaled = await generateUpscale(projectId, outputFile, upscaleResolution, onLog);
1140
+ if ("error" in upscaled)
1141
+ return { error: upscaled.error };
1142
+ return upscaled;
1143
+ }
1144
+ return {
1145
+ filename: outputFile,
1146
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
1147
+ };
1148
+ }
1149
+ finally {
1150
+ for (const path of tempImagePaths) {
1151
+ try {
1152
+ unlinkSync(path);
1153
+ }
1154
+ catch {
1155
+ // already removed
1156
+ }
1157
+ }
1158
+ try {
1159
+ rmSync(tempDir, { force: true });
1160
+ }
1161
+ catch {
1162
+ // ignore cleanup failures
1163
+ }
1164
+ }
1165
+ }
628
1166
  // ========== Routes ==========
629
1167
  export async function renderMediaRoutes({ app, getUvPath, }) {
630
1168
  // ========== Upload ==========
@@ -909,6 +1447,60 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
909
1447
  raw.sort((a, b) => b.birthtime - a.birthtime);
910
1448
  res.json(raw.map(({ filename, url }) => ({ filename, url })));
911
1449
  });
1450
+ // List previously generated voice clones by scanning the voices folder. Each
1451
+ // voice lives in its own subfolder with a meta.json (no central JSON index).
1452
+ app.get("/api/projects/:id/voices", (req, res) => {
1453
+ const { id } = req.params;
1454
+ if (!isValidProjectId(id)) {
1455
+ res.status(400).json({ error: "Invalid project ID" });
1456
+ return;
1457
+ }
1458
+ const voicesRoot = join(OUTPUT_DIR, id, "voices");
1459
+ const results = [];
1460
+ if (existsSync(voicesRoot)) {
1461
+ let names = [];
1462
+ try {
1463
+ names = readdirSync(voicesRoot);
1464
+ }
1465
+ catch {
1466
+ names = [];
1467
+ }
1468
+ for (const name of names) {
1469
+ const voiceDir = join(voicesRoot, name);
1470
+ let isDir = false;
1471
+ try {
1472
+ isDir = statSync(voiceDir).isDirectory();
1473
+ }
1474
+ catch {
1475
+ isDir = false;
1476
+ }
1477
+ if (!isDir)
1478
+ continue;
1479
+ let meta = null;
1480
+ try {
1481
+ meta = JSON.parse(readFileSync(join(voiceDir, "meta.json"), "utf-8"));
1482
+ }
1483
+ catch {
1484
+ continue; // folder without a meta.json — skip
1485
+ }
1486
+ const filename = String(meta?.filename || TTS_OUTPUT_FILENAME);
1487
+ const audioPath = join(voiceDir, filename);
1488
+ if (!existsSync(audioPath))
1489
+ continue;
1490
+ results.push({
1491
+ id: name,
1492
+ transcript: String(meta?.transcript ?? ""),
1493
+ quality: String(meta?.quality ?? "high"),
1494
+ refAudioFilename: meta?.refAudioFilename ?? null,
1495
+ filename,
1496
+ createdAt: meta?.createdAt ?? null,
1497
+ url: `/api/files?path=${encodeURIComponent(audioPath)}`,
1498
+ });
1499
+ }
1500
+ }
1501
+ results.sort((a, b) => String(b.createdAt ?? "").localeCompare(String(a.createdAt ?? "")));
1502
+ res.json(results);
1503
+ });
912
1504
  // ========== Render: Text-to-Image ==========
913
1505
  app.post("/api/render/text-to-image", async (req, res) => {
914
1506
  const { prompt, projectId, width = 512, height = 512, device = "mps", } = req.body || {};
@@ -1002,7 +1594,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1002
1594
  });
1003
1595
  // ========== Render: Image-to-Video ==========
1004
1596
  app.post("/api/render/image-to-video", async (req, res) => {
1005
- const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", } = req.body || {};
1597
+ const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", stage1Steps, stage2Steps, model, } = req.body || {};
1006
1598
  if (!prompt) {
1007
1599
  res.status(400).json({ error: "Prompt is required" });
1008
1600
  return;
@@ -1058,6 +1650,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1058
1650
  const videoHeight = Number(height) || 480;
1059
1651
  const videoFrames = Number(frames) || 121;
1060
1652
  const videoFps = Number(frameRate) || 24;
1653
+ const stage1 = Math.max(1, Math.round(Number(stage1Steps)) || 30);
1654
+ const stage2 = Math.max(1, Math.round(Number(stage2Steps)) || 3);
1061
1655
  send("progress", {
1062
1656
  status: "starting",
1063
1657
  label: "Generating video...",
@@ -1075,10 +1669,14 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1075
1669
  "ltx-2-mlx",
1076
1670
  "generate",
1077
1671
  "--model",
1078
- "dgrauet/ltx-2.3-mlx-q8",
1672
+ resolveLtxModel(model),
1079
1673
  "--prompt",
1080
1674
  prompt,
1081
1675
  stageFlagFor(mode),
1676
+ "--stage1-steps",
1677
+ String(stage1),
1678
+ "--stage2-steps",
1679
+ String(stage2),
1082
1680
  // "--low-ram",
1083
1681
  "--frames",
1084
1682
  String(videoFrames),
@@ -1674,252 +2272,19 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1674
2272
  res.end();
1675
2273
  }
1676
2274
  });
1677
- // ========== Render: Extend-Video ==========
1678
- app.post("/api/render/extend-video", async (req, res) => {
1679
- const { prompt, videoPath, projectId, extendFrames = 2 } = req.body || {};
1680
- if (!prompt) {
1681
- res.status(400).json({ error: "Prompt is required" });
1682
- return;
1683
- }
1684
- if (!videoPath) {
1685
- res.status(400).json({ error: "Video path is required" });
1686
- return;
2275
+ // ========== MLX-Audio: Status ==========
2276
+ app.get("/api/mlxaudio/status", async (_req, res) => {
2277
+ // Never throw on a missing uv installation state is based on the folder.
2278
+ try {
2279
+ await getUvPath();
1687
2280
  }
1688
- if (!projectId) {
1689
- res.status(400).json({ error: "Project ID is required" });
1690
- return;
1691
- }
1692
- // Resolve video path — only bare .mp4 filenames in this project's output dir
1693
- const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
1694
- if (!resolvedVideo) {
1695
- res.status(400).json({
1696
- error: "Invalid video path. Provide a filename previously generated in this project.",
1697
- });
1698
- return;
1699
- }
1700
- // SSE headers
1701
- res.writeHead(200, {
1702
- "Content-Type": "text/event-stream",
1703
- "Cache-Control": "no-cache",
1704
- Connection: "keep-alive",
1705
- "X-Accel-Buffering": "no",
1706
- });
1707
- const send = (event, data) => {
1708
- res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1709
- };
1710
- try {
1711
- const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
1712
- if (!existsSync(ltxFolder)) {
1713
- send("error", { error: "ltx-2-mlx not found. Run setup first." });
1714
- res.end();
1715
- return;
1716
- }
1717
- const uvPath = await getUvPath();
1718
- const projectOutputDir = join(OUTPUT_DIR, projectId);
1719
- ensureDir(projectOutputDir);
1720
- const outputFile = `extended-${Date.now()}.mp4`;
1721
- const outputPath = join(projectOutputDir, outputFile);
1722
- const framesToAdd = Number(extendFrames) || 2;
1723
- send("progress", {
1724
- status: "starting",
1725
- label: "Extending video...",
1726
- inputFile: resolvedVideo,
1727
- outputFile,
1728
- settings: { extendFrames: framesToAdd },
1729
- });
1730
- const proc = spawn([
1731
- uvPath,
1732
- "run",
1733
- "ltx-2-mlx",
1734
- "extend",
1735
- "--model",
1736
- "dgrauet/ltx-2.3-mlx-q8",
1737
- "--prompt",
1738
- prompt,
1739
- "--video",
1740
- resolvedVideo,
1741
- "--extend-frames",
1742
- String(framesToAdd),
1743
- "--output",
1744
- outputPath,
1745
- ], {
1746
- env: process.env,
1747
- cwd: ltxFolder,
1748
- stdout: "pipe",
1749
- stderr: "pipe",
1750
- });
1751
- activeProc = proc;
1752
- // Stream stdout/stderr concurrently
1753
- const stdoutPromise = streamToSSE(proc.stdout, "Extend", send);
1754
- const stderrText = await streamToSSE(proc.stderr, "Extend", send);
1755
- await stdoutPromise;
1756
- const exitCode = await proc.exited;
1757
- const success = exitCode === 0 && existsSync(outputPath);
1758
- if (success) {
1759
- send("complete", {
1760
- success: true,
1761
- path: outputPath,
1762
- filename: outputFile,
1763
- });
1764
- }
1765
- else {
1766
- send("error", {
1767
- error: stderrText || `Process exited with code ${exitCode}`,
1768
- exitCode,
1769
- });
1770
- }
1771
- }
1772
- catch (e) {
1773
- send("error", { error: String(e) });
1774
- }
1775
- finally {
1776
- activeProc = null;
1777
- res.end();
1778
- }
1779
- });
1780
- // ========== MLX-Audio: Status ==========
1781
- app.get("/api/mlxaudio/status", async (_req, res) => {
1782
- // Never throw on a missing uv — installation state is based on the folder.
1783
- try {
1784
- await getUvPath();
1785
- }
1786
- catch {
1787
- // uv not found; installed is still reported from the folder below.
2281
+ catch {
2282
+ // uv not found; installed is still reported from the folder below.
1788
2283
  }
1789
2284
  res.json({
1790
2285
  installed: existsSync(join(PYTHON_DIR, "mlx-audio")),
1791
2286
  });
1792
2287
  });
1793
- // ========== Render: Text-to-Speech (mlx-audio) ==========
1794
- app.post("/api/render/tts", async (req, res) => {
1795
- const { text, refAudioPath, projectId, outputDir, quality, voiceId } = req.body || {};
1796
- if (!text || typeof text !== "string" || !text.trim()) {
1797
- res.status(400).json({ error: "Text is required" });
1798
- return;
1799
- }
1800
- if (!refAudioPath) {
1801
- res.status(400).json({ error: "Reference audio is required" });
1802
- return;
1803
- }
1804
- if (!projectId || !isValidProjectId(String(projectId))) {
1805
- res.status(400).json({ error: "Invalid project ID" });
1806
- return;
1807
- }
1808
- if (quality !== "low" && quality !== "high") {
1809
- res.status(400).json({ error: "Quality must be 'low' or 'high'" });
1810
- return;
1811
- }
1812
- // Resolve reference audio — only bare filenames in this project's dirs
1813
- const resolvedRef = resolveSafePath(refAudioPath, projectId);
1814
- if (!resolvedRef) {
1815
- res.status(400).json({
1816
- error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
1817
- });
1818
- return;
1819
- }
1820
- // SSE headers
1821
- res.writeHead(200, {
1822
- "Content-Type": "text/event-stream",
1823
- "Cache-Control": "no-cache",
1824
- Connection: "keep-alive",
1825
- "X-Accel-Buffering": "no",
1826
- });
1827
- const send = (event, data) => {
1828
- res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1829
- };
1830
- try {
1831
- const uvPath = await getUvPath();
1832
- const projectOutputDir = resolveOutputDir(outputDir, projectId);
1833
- if (!projectOutputDir) {
1834
- send("error", { error: "Invalid output directory." });
1835
- res.end();
1836
- return;
1837
- }
1838
- // TTS audio is saved under <projectOutputDir>/voices/<voiceId>/ so each
1839
- // row's voiceover stays isolated in its own folder. Sanitize voiceId to
1840
- // alphanumerics/_/- so it can never escape the voices directory.
1841
- const safeVoiceId = String(voiceId ?? `voice-${Date.now()}`)
1842
- .replace(/[^a-zA-Z0-9_-]/g, "_")
1843
- .slice(0, 64);
1844
- const voiceDir = join(projectOutputDir, "voices", safeVoiceId);
1845
- ensureDir(voiceDir);
1846
- // Regenerating a row must overwrite the previous clip: delete any
1847
- // existing output so the generator writes a fresh audio_000.mp3.
1848
- const prevAudio = join(voiceDir, TTS_OUTPUT_FILENAME);
1849
- if (existsSync(prevAudio)) {
1850
- try {
1851
- unlinkSync(prevAudio);
1852
- }
1853
- catch {
1854
- // Ignore — the generator will overwrite the file regardless.
1855
- }
1856
- }
1857
- send("progress", {
1858
- status: "starting",
1859
- label: "Generating speech...",
1860
- model: TTS_MODELS[quality],
1861
- });
1862
- // No --play flag: this is a silent server-side batch generation. The mp3
1863
- // lands in <projectOutputDir>/voices/<voiceId>/, which is servable via
1864
- // /api/files and resolvable via resolveSafePath for the mux step.
1865
- const proc = spawn([
1866
- uvPath,
1867
- "run",
1868
- "mlx_audio.tts.generate",
1869
- "--model",
1870
- TTS_MODELS[quality],
1871
- "--text",
1872
- text,
1873
- "--ref_audio",
1874
- resolvedRef,
1875
- "--output",
1876
- voiceDir,
1877
- "--audio_format",
1878
- "mp3",
1879
- "--play",
1880
- "--instruct",
1881
- "slow down speech",
1882
- ], {
1883
- env: process.env,
1884
- cwd: voiceDir,
1885
- stdout: "pipe",
1886
- stderr: "pipe",
1887
- });
1888
- activeProc = proc;
1889
- const stdoutPromise = streamToSSE(proc.stdout, "TTS", send);
1890
- const stderrText = await streamToSSE(proc.stderr, "TTS", send);
1891
- await stdoutPromise;
1892
- const exitCode = await proc.exited;
1893
- if (exitCode === 0) {
1894
- const path = resolveAudioFile(voiceDir);
1895
- if (path) {
1896
- send("complete", {
1897
- success: true,
1898
- path,
1899
- filename: path.split(sep).pop(),
1900
- });
1901
- }
1902
- else {
1903
- send("error", {
1904
- error: "TTS completed but no audio file was produced",
1905
- });
1906
- }
1907
- }
1908
- else {
1909
- send("error", {
1910
- error: stderrText || `Process exited with code ${exitCode}`,
1911
- exitCode,
1912
- });
1913
- }
1914
- }
1915
- catch (e) {
1916
- send("error", { error: String(e) });
1917
- }
1918
- finally {
1919
- activeProc = null;
1920
- res.end();
1921
- }
1922
- });
1923
2288
  // ========== Render: Voice Chat (TTS with reference voice) ==========
1924
2289
  app.post("/api/render/voice-chat", async (req, res) => {
1925
2290
  const { text, refAudioPath, projectId } = req.body || {};
@@ -2029,38 +2394,17 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2029
2394
  res.end();
2030
2395
  }
2031
2396
  });
2032
- // ========== Render: Mux Video + Audio ==========
2033
- app.post("/api/render/mux-audio", async (req, res) => {
2034
- const { videoPath, audioPath, projectId, outputDir } = req.body || {};
2035
- if (!videoPath) {
2036
- res.status(400).json({ error: "Video path is required" });
2037
- return;
2038
- }
2039
- if (!audioPath) {
2040
- res.status(400).json({ error: "Audio path is required" });
2041
- return;
2042
- }
2043
- if (!projectId || !isValidProjectId(String(projectId))) {
2044
- res.status(400).json({ error: "Invalid project ID" });
2045
- return;
2046
- }
2047
- // Resolve video path — only bare .mp4 filenames in this project's output dir
2048
- const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
2049
- if (!resolvedVideo) {
2050
- res.status(400).json({
2051
- error: "Invalid video path. Provide a filename previously generated in this project.",
2052
- });
2053
- return;
2054
- }
2055
- // Resolve audio path — only bare filenames in this project's dirs
2056
- const resolvedAudio = resolveSafePath(audioPath, projectId);
2057
- if (!resolvedAudio) {
2058
- res.status(400).json({
2059
- error: "Invalid audio path. Provide a filename previously generated or uploaded to this project.",
2060
- });
2061
- return;
2062
- }
2063
- // SSE headers
2397
+ // ========== MLX-Gen: Status ==========
2398
+ app.get("/api/mlxgen/status", (_req, res) => {
2399
+ res.json({
2400
+ installed: isMlxgenInstalled(),
2401
+ zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
2402
+ fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
2403
+ seedvr2Downloaded: isModelDownloaded(SEEDVR2_MODEL),
2404
+ });
2405
+ });
2406
+ // ========== MLX-Gen: Install ==========
2407
+ app.post("/api/mlxgen/install", async (_req, res) => {
2064
2408
  res.writeHead(200, {
2065
2409
  "Content-Type": "text/event-stream",
2066
2410
  "Cache-Control": "no-cache",
@@ -2071,54 +2415,22 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2071
2415
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2072
2416
  };
2073
2417
  try {
2074
- const ffmpegBin = await getFfmpegBin();
2075
- const projectOutputDir = resolveOutputDir(outputDir, projectId);
2076
- if (!projectOutputDir) {
2077
- send("error", { error: "Invalid output directory." });
2078
- res.end();
2079
- return;
2080
- }
2081
- const finalFile = `voice-${Date.now()}.mp4`;
2082
- const finalPath = join(projectOutputDir, finalFile);
2418
+ const uvPath = await getUvPath();
2083
2419
  send("progress", {
2084
2420
  status: "starting",
2085
- label: "Muxing video and audio...",
2086
- outputFile: finalFile,
2421
+ label: "Installing mlx-gen...",
2087
2422
  });
2088
- const proc = spawn([
2089
- ffmpegBin,
2090
- "-y",
2091
- "-i",
2092
- resolvedVideo,
2093
- "-i",
2094
- resolvedAudio,
2095
- "-map",
2096
- "0:v",
2097
- "-map",
2098
- "1:a",
2099
- "-c:v",
2100
- "copy",
2101
- "-c:a",
2102
- "aac",
2103
- "-b:a",
2104
- "192k",
2105
- "-shortest",
2106
- finalPath,
2107
- ], {
2423
+ const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
2108
2424
  stdout: "pipe",
2109
2425
  stderr: "pipe",
2110
2426
  });
2111
2427
  activeProc = proc;
2112
- const stdoutPromise = streamToSSE(proc.stdout, "Mux", send);
2113
- const stderrText = await streamToSSE(proc.stderr, "Mux", send);
2428
+ const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
2429
+ const stderrText = await streamToSSE(proc.stderr, "Install", send);
2114
2430
  await stdoutPromise;
2115
2431
  const exitCode = await proc.exited;
2116
- if (exitCode === 0 && existsSync(finalPath)) {
2117
- send("complete", {
2118
- success: true,
2119
- path: finalPath,
2120
- filename: finalFile,
2121
- });
2432
+ if (exitCode === 0) {
2433
+ send("complete", { success: true });
2122
2434
  }
2123
2435
  else {
2124
2436
  send("error", {
@@ -2135,16 +2447,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2135
2447
  res.end();
2136
2448
  }
2137
2449
  });
2138
- // ========== MLX-Gen: Status ==========
2139
- app.get("/api/mlxgen/status", (_req, res) => {
2140
- res.json({
2141
- installed: isMlxgenInstalled(),
2142
- zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
2143
- fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
2144
- });
2145
- });
2146
- // ========== MLX-Gen: Install ==========
2147
- app.post("/api/mlxgen/install", async (_req, res) => {
2450
+ // ========== MLX-Gen: Download Z-Image Model ==========
2451
+ app.post("/api/mlxgen/download-z-model", async (_req, res) => {
2148
2452
  res.writeHead(200, {
2149
2453
  "Content-Type": "text/event-stream",
2150
2454
  "Cache-Control": "no-cache",
@@ -2154,19 +2458,20 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2154
2458
  const send = (event, data) => {
2155
2459
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2156
2460
  };
2461
+ const model = Z_IMAGE_MODEL;
2157
2462
  try {
2158
- const uvPath = await getUvPath();
2463
+ const mlxgen = await getMlxgenBin();
2159
2464
  send("progress", {
2160
2465
  status: "starting",
2161
- label: "Installing mlx-gen...",
2466
+ label: `Downloading model ${model}...`,
2162
2467
  });
2163
- const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
2468
+ const proc = spawn([mlxgen, "download", "--model", model], {
2164
2469
  stdout: "pipe",
2165
2470
  stderr: "pipe",
2166
2471
  });
2167
2472
  activeProc = proc;
2168
- const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
2169
- const stderrText = await streamToSSE(proc.stderr, "Install", send);
2473
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2474
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2170
2475
  await stdoutPromise;
2171
2476
  const exitCode = await proc.exited;
2172
2477
  if (exitCode === 0) {
@@ -2187,8 +2492,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2187
2492
  res.end();
2188
2493
  }
2189
2494
  });
2190
- // ========== MLX-Gen: Download Z-Image Model ==========
2191
- app.post("/api/mlxgen/download-z-model", async (_req, res) => {
2495
+ // ========== MLX-Gen: Download FLUX.2 Klein Model ==========
2496
+ app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
2192
2497
  res.writeHead(200, {
2193
2498
  "Content-Type": "text/event-stream",
2194
2499
  "Cache-Control": "no-cache",
@@ -2198,14 +2503,13 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2198
2503
  const send = (event, data) => {
2199
2504
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2200
2505
  };
2201
- const model = Z_IMAGE_MODEL;
2202
2506
  try {
2203
2507
  const mlxgen = await getMlxgenBin();
2204
2508
  send("progress", {
2205
2509
  status: "starting",
2206
- label: `Downloading model ${model}...`,
2510
+ label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
2207
2511
  });
2208
- const proc = spawn([mlxgen, "download", "--model", model], {
2512
+ const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
2209
2513
  stdout: "pipe",
2210
2514
  stderr: "pipe",
2211
2515
  });
@@ -2232,8 +2536,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2232
2536
  res.end();
2233
2537
  }
2234
2538
  });
2235
- // ========== MLX-Gen: Download FLUX.2 Klein Model ==========
2236
- app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
2539
+ // ========== MLX-Gen: Download SeedVR2 Model ==========
2540
+ app.post("/api/mlxgen/download-seedvr2-model", async (_req, res) => {
2237
2541
  res.writeHead(200, {
2238
2542
  "Content-Type": "text/event-stream",
2239
2543
  "Cache-Control": "no-cache",
@@ -2247,9 +2551,9 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2247
2551
  const mlxgen = await getMlxgenBin();
2248
2552
  send("progress", {
2249
2553
  status: "starting",
2250
- label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
2554
+ label: `Downloading model ${SEEDVR2_MODEL}...`,
2251
2555
  });
2252
- const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
2556
+ const proc = spawn([mlxgen, "download", "--model", SEEDVR2_MODEL], {
2253
2557
  stdout: "pipe",
2254
2558
  stderr: "pipe",
2255
2559
  });
@@ -2285,10 +2589,63 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2285
2589
  res.json({
2286
2590
  installed: whichSync("hf") !== null,
2287
2591
  ltxDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx-q8"),
2592
+ ltxBaseDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx"),
2288
2593
  ttsDownloaded: isModelDownloaded("Qwen/Qwen3-TTS-12Hz-1.7B-Base"),
2289
2594
  mlxVlmDownloaded: isModelDownloaded(MLX_VLM_MODEL),
2290
2595
  });
2291
2596
  });
2597
+ // ========== Dots-TTS: Status + Model Download ==========
2598
+ app.get("/api/dots-tts/status", (_req, res) => {
2599
+ res.json({ downloaded: isDotsTtsModelDownloaded() });
2600
+ });
2601
+ app.post("/api/dots-tts/download-model", async (_req, res) => {
2602
+ res.writeHead(200, {
2603
+ "Content-Type": "text/event-stream",
2604
+ "Cache-Control": "no-cache",
2605
+ Connection: "keep-alive",
2606
+ "X-Accel-Buffering": "no",
2607
+ });
2608
+ const send = (event, data) => {
2609
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2610
+ };
2611
+ try {
2612
+ ensureDir(DOTS_TTS_FOLDER);
2613
+ send("progress", {
2614
+ status: "starting",
2615
+ label: "Downloading dots-tts model (mf-int4)...",
2616
+ });
2617
+ const proc = spawn([
2618
+ "hf",
2619
+ "download",
2620
+ "shraey/dots-tts-mlx",
2621
+ "--include",
2622
+ "mf-int4/*",
2623
+ "--local-dir",
2624
+ "./dots-tts-mlx-weights",
2625
+ ], { cwd: DOTS_TTS_FOLDER, stdout: "pipe", stderr: "pipe" });
2626
+ activeProc = proc;
2627
+ const stdoutPromise = streamToSSE(proc.stdout, "dots-tts", send);
2628
+ const stderrText = await streamToSSE(proc.stderr, "dots-tts", send);
2629
+ await stdoutPromise;
2630
+ const exitCode = await proc.exited;
2631
+ if (exitCode === 0) {
2632
+ send("complete", { success: true });
2633
+ }
2634
+ else {
2635
+ send("error", {
2636
+ error: stderrText || `Process exited with code ${exitCode}`,
2637
+ exitCode,
2638
+ });
2639
+ }
2640
+ }
2641
+ catch (e) {
2642
+ send("error", { error: String(e) });
2643
+ }
2644
+ finally {
2645
+ activeProc = null;
2646
+ res.end();
2647
+ }
2648
+ });
2292
2649
  app.post("/api/hf/install", async (_req, res) => {
2293
2650
  res.writeHead(200, {
2294
2651
  "Content-Type": "text/event-stream",
@@ -2370,6 +2727,48 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2370
2727
  res.end();
2371
2728
  }
2372
2729
  });
2730
+ app.post("/api/hf/download-ltx-base", async (_req, res) => {
2731
+ res.writeHead(200, {
2732
+ "Content-Type": "text/event-stream",
2733
+ "Cache-Control": "no-cache",
2734
+ Connection: "keep-alive",
2735
+ "X-Accel-Buffering": "no",
2736
+ });
2737
+ const send = (event, data) => {
2738
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2739
+ };
2740
+ try {
2741
+ send("progress", {
2742
+ status: "starting",
2743
+ label: "Downloading dgrauet/ltx-2.3-mlx...",
2744
+ });
2745
+ const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx"], {
2746
+ stdout: "pipe",
2747
+ stderr: "pipe",
2748
+ });
2749
+ activeProc = proc;
2750
+ const stdoutPromise = streamToSSE(proc.stdout, "HF Download", send);
2751
+ const stderrText = await streamToSSE(proc.stderr, "HF Download", send);
2752
+ await stdoutPromise;
2753
+ const exitCode = await proc.exited;
2754
+ if (exitCode === 0) {
2755
+ send("complete", { success: true });
2756
+ }
2757
+ else {
2758
+ send("error", {
2759
+ error: stderrText || `Process exited with code ${exitCode}`,
2760
+ exitCode,
2761
+ });
2762
+ }
2763
+ }
2764
+ catch (e) {
2765
+ send("error", { error: String(e) });
2766
+ }
2767
+ finally {
2768
+ activeProc = null;
2769
+ res.end();
2770
+ }
2771
+ });
2373
2772
  app.post("/api/hf/download-mlx-vlm", async (_req, res) => {
2374
2773
  res.writeHead(200, {
2375
2774
  "Content-Type": "text/event-stream",
@@ -2977,109 +3376,6 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2977
3376
  writeCharacters(characters);
2978
3377
  res.json(removed);
2979
3378
  });
2980
- // Save an extracted video frame into the project's extracted-frames folder.
2981
- app.post("/api/extracted-frames", (req, res) => {
2982
- const { image, filename, projectId } = req.body || {};
2983
- if (!image) {
2984
- res.status(400).json({ error: "Image data is required (base64)" });
2985
- return;
2986
- }
2987
- if (!projectId || !isValidProjectId(String(projectId))) {
2988
- res.status(400).json({ error: "Invalid project ID" });
2989
- return;
2990
- }
2991
- try {
2992
- const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
2993
- const buffer = Buffer.from(base64, "base64");
2994
- const dir = join(EXTRACTED_FRAMES_DIR, String(projectId));
2995
- ensureDir(dir);
2996
- const safeName = (filename || `frame-${Date.now()}.png`).replace(/[^a-zA-Z0-9._-]/g, "_");
2997
- writeFileSync(join(dir, safeName), buffer);
2998
- res.json({
2999
- success: true,
3000
- path: join(dir, safeName),
3001
- filename: safeName,
3002
- size: buffer.length,
3003
- });
3004
- }
3005
- catch (e) {
3006
- res
3007
- .status(500)
3008
- .json({ error: "Failed to save frame", details: String(e) });
3009
- }
3010
- });
3011
- // Save the character sheet: current.png plus a backup copy under backup/.
3012
- app.post("/api/character-sheet", (req, res) => {
3013
- const { image, projectId } = req.body || {};
3014
- if (!image) {
3015
- res.status(400).json({ error: "Image data is required (base64)" });
3016
- return;
3017
- }
3018
- if (!projectId || !isValidProjectId(String(projectId))) {
3019
- res.status(400).json({ error: "Invalid project ID" });
3020
- return;
3021
- }
3022
- try {
3023
- const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
3024
- const buffer = Buffer.from(base64, "base64");
3025
- const projectDir = join(CHARACTER_SHEET_DIR, String(projectId));
3026
- const backupDir = join(projectDir, "backup");
3027
- ensureDir(backupDir);
3028
- const backupId = randomUUID();
3029
- const backupPath = join(backupDir, `${backupId}.png`);
3030
- const currentPath = join(projectDir, "current.png");
3031
- writeFileSync(backupPath, buffer);
3032
- writeFileSync(currentPath, buffer);
3033
- res.json({
3034
- success: true,
3035
- path: currentPath,
3036
- backupPath,
3037
- backupFilename: `${backupId}.png`,
3038
- size: buffer.length,
3039
- });
3040
- }
3041
- catch (e) {
3042
- res
3043
- .status(500)
3044
- .json({ error: "Failed to save character sheet", details: String(e) });
3045
- }
3046
- });
3047
- // List saved character sheet images (current.png) for a project.
3048
- app.get("/api/projects/:id/character-sheets", (req, res) => {
3049
- const { id } = req.params;
3050
- if (!isValidProjectId(id)) {
3051
- res.status(400).json({ error: "Invalid project ID" });
3052
- return;
3053
- }
3054
- const projectDir = join(CHARACTER_SHEET_DIR, id);
3055
- const results = [];
3056
- if (existsSync(projectDir)) {
3057
- let entries = [];
3058
- try {
3059
- entries = readdirSync(projectDir);
3060
- }
3061
- catch {
3062
- entries = [];
3063
- }
3064
- for (const entry of entries) {
3065
- if (!entry.toLowerCase().endsWith(".png"))
3066
- continue;
3067
- const fullPath = join(projectDir, entry);
3068
- try {
3069
- if (!statSync(fullPath).isFile())
3070
- continue;
3071
- }
3072
- catch {
3073
- continue;
3074
- }
3075
- results.push({
3076
- filename: entry,
3077
- url: `/api/files?path=${encodeURIComponent(fullPath)}`,
3078
- });
3079
- }
3080
- }
3081
- res.json(results);
3082
- });
3083
3379
  // Open project folder in Finder
3084
3380
  app.post("/api/projects/:id/open-folder", (req, res) => {
3085
3381
  const { type } = req.body || {};