@effectnode/media 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/backend/movie-backend/agent/prompt/ltx.txt +20 -0
  2. package/dist/backend/movie-backend/agent/prompt/script.md +111 -1
  3. package/dist/backend/movie-backend/core.js +41 -0
  4. package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
  5. package/dist/backend/movie-backend/generation-queue.js +75 -8
  6. package/dist/backend/movie-backend/render-media.d.ts +79 -2
  7. package/dist/backend/movie-backend/render-media.js +694 -418
  8. package/frontend/src/movie-app/components/EditorTabs/AdvancedVoiceCloneTab.tsx +366 -0
  9. package/frontend/src/movie-app/components/EditorTabs/AudioToVideoTab.tsx +406 -0
  10. package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +47 -0
  11. package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +155 -1
  12. package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +33 -13
  13. package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +26 -5
  14. package/frontend/src/movie-app/components/EditorTabs/UpscaleTab.tsx +343 -0
  15. package/frontend/src/movie-app/components/EditorTabs/VoiceCloneTab.tsx +365 -0
  16. package/frontend/src/movie-app/components/ProjectEditorPage.tsx +130 -125
  17. package/frontend/src/movie-app/stores/advancedVoiceCloneStore.ts +205 -0
  18. package/frontend/src/movie-app/stores/aiModelStore.ts +25 -2
  19. package/frontend/src/movie-app/stores/audioToVideoStore.ts +274 -0
  20. package/frontend/src/movie-app/stores/generationStore.ts +156 -255
  21. package/frontend/src/movie-app/stores/movieStudioStore.ts +10 -3
  22. package/frontend/src/movie-app/stores/queueStore.ts +47 -1
  23. package/frontend/src/movie-app/stores/upscaleStore.ts +118 -0
  24. package/frontend/src/movie-app/stores/voiceCloneStore.ts +227 -0
  25. package/package.json +1 -1
  26. package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +0 -913
  27. package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +0 -305
  28. package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +0 -249
  29. package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +0 -267
  30. package/frontend/src/movie-app/lib/batchVoiceStorage.ts +0 -75
  31. package/frontend/src/movie-app/stores/batchVoiceStore.ts +0 -990
  32. package/frontend/src/movie-app/stores/sceneVisualStore.ts +0 -251
@@ -1,6 +1,6 @@
1
1
  import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync, readdirSync, statSync, unlinkSync, rmSync, copyFileSync, } from "node:fs";
2
2
  import { homedir } from "node:os";
3
- import { join, sep } from "node:path";
3
+ import { dirname, join, sep } from "node:path";
4
4
  import { spawn, whichSync, mimeType } from "./process.js";
5
5
  // Track the currently active spawn process so it can be cancelled
6
6
  let activeProc = null;
@@ -16,7 +16,6 @@ const APP_DATA_DIR = join(homedir(), "media-studio");
16
16
  const OUTPUT_DIR = join(APP_DATA_DIR, "output");
17
17
  const UPLOAD_DIR = join(APP_DATA_DIR, "upload");
18
18
  const AGENT_UPLOAD_DIR = join(APP_DATA_DIR, "agent-upload");
19
- const EXTRACTED_FRAMES_DIR = join(APP_DATA_DIR, "extracted-frames");
20
19
  const AGENTS_DIR = join(APP_DATA_DIR, "agents");
21
20
  const JSON_DIR = join(APP_DATA_DIR, "json");
22
21
  const PYTHON_DIR = join(APP_DATA_DIR, "python-src");
@@ -25,7 +24,10 @@ const PROJECTS_FILE = join(JSON_DIR, "projects.json");
25
24
  const CHARACTERS_FILE = join(JSON_DIR, "characters.json");
26
25
  const Z_IMAGE_MODEL = "AbstractFramework/z-image-turbo-8bit";
27
26
  const FLUX_KLEIN_MODEL = "AbstractFramework/flux.2-klein-4b-8bit";
27
+ const SEEDVR2_MODEL = "AbstractFramework/seedvr2-7b-8bit";
28
28
  const MLX_VLM_MODEL = "mlx-community/gemma-4-e4b-it-8bit";
29
+ const LTX_MODEL_HIGH_QUALITY = "dgrauet/ltx-2.3-mlx";
30
+ const LTX_MODEL_STANDARD = "dgrauet/ltx-2.3-mlx-q8";
29
31
  const VIDEO_STAGE_FLAGS = {
30
32
  distilled: "--distilled",
31
33
  "one-stage": "--one-stage",
@@ -41,6 +43,12 @@ function stageFlagFor(mode) {
41
43
  ? (VIDEO_STAGE_FLAGS[mode] ?? "--distilled")
42
44
  : "--distilled";
43
45
  }
46
+ /** Resolve a ltx-2-mlx model id, falling back to the q8 standard model. */
47
+ function resolveLtxModel(model) {
48
+ return String(model) === LTX_MODEL_HIGH_QUALITY
49
+ ? LTX_MODEL_HIGH_QUALITY
50
+ : LTX_MODEL_STANDARD;
51
+ }
44
52
  // ========== Project Helpers ==========
45
53
  function ensureDir(dir) {
46
54
  if (!existsSync(dir)) {
@@ -187,22 +195,6 @@ function resolveSafePath(candidate, projectId) {
187
195
  }
188
196
  return null;
189
197
  }
190
- /** Resolve and validate a user-supplied video filename. Only bare .mp4 names in the output dir. */
191
- function resolveSafeVideoPath(candidate, projectId) {
192
- const base = candidate.split(/[/\\]/).pop() || candidate;
193
- if (base !== candidate || base.includes("..") || base.startsWith(".")) {
194
- return null;
195
- }
196
- if (!base.toLowerCase().endsWith(".mp4"))
197
- return null;
198
- const candidatePath = join(OUTPUT_DIR, projectId, base);
199
- if (!existsSync(candidatePath))
200
- return null;
201
- const resolved = realpathSync(candidatePath);
202
- if (isPathAllowed(resolved))
203
- return resolved;
204
- return null;
205
- }
206
198
  /**
207
199
  * Resolve the `mlxgen` executable installed via `uv tool install --upgrade mlx-gen`.
208
200
  * uv tool installs binaries into `~/.local/bin`; fall back to relying on PATH.
@@ -219,6 +211,50 @@ async function getMlxgenBin() {
219
211
  }
220
212
  return "mlxgen";
221
213
  }
214
+ /** Resolve the `dots-tts` executable installed via `uv tool install dots-tts`. */
215
+ async function getDotsTtsBin() {
216
+ const candidates = [
217
+ join(homedir(), ".local", "bin", "dots-tts"),
218
+ "/opt/homebrew/bin/dots-tts",
219
+ "/usr/local/bin/dots-tts",
220
+ ];
221
+ for (const p of candidates) {
222
+ if (existsSync(p))
223
+ return p;
224
+ }
225
+ return "dots-tts";
226
+ }
227
+ /** The cloned dots-tts-mlx project directory. */
228
+ const DOTS_TTS_FOLDER = join(APP_DATA_DIR, "python-src", "dots-tts-mlx");
229
+ /** Base directory where dots-tts MLX weights live (inside the project folder). */
230
+ const DOTS_TTS_WEIGHTS_DIR = join(DOTS_TTS_FOLDER, "dots-tts-mlx-weights");
231
+ /**
232
+ * Resolve a dots-tts `--model` value. Accepts `./dots-tts-mlx-weights/int4`
233
+ * (relative to the dots-tts-mlx folder), a bare variant (`int4`), or an
234
+ * explicit path.
235
+ */
236
+ function resolveDotsTtsModel(model) {
237
+ if (!model)
238
+ return join(DOTS_TTS_WEIGHTS_DIR, "int4");
239
+ if (model.startsWith("./"))
240
+ return join(DOTS_TTS_FOLDER, model.slice(2));
241
+ if (model.includes("/"))
242
+ return model;
243
+ return join(DOTS_TTS_WEIGHTS_DIR, model);
244
+ }
245
+ /** True when the dots-tts `int4` weights have been downloaded. */
246
+ function isDotsTtsModelDownloaded() {
247
+ return existsSync(join(DOTS_TTS_WEIGHTS_DIR, "int4"));
248
+ }
249
+ /** Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel). */
250
+ async function getFfmpegBin() {
251
+ const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
252
+ for (const p of candidates) {
253
+ if (existsSync(p))
254
+ return p;
255
+ }
256
+ return "ffmpeg";
257
+ }
222
258
  /** True when the `mlxgen` executable is installed (known paths or PATH). */
223
259
  function isMlxgenInstalled() {
224
260
  const candidates = [
@@ -293,18 +329,6 @@ function isMlxVlmInstalled() {
293
329
  return false;
294
330
  }
295
331
  }
296
- /**
297
- * Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel).
298
- * Falls back to relying on PATH when neither known location exists.
299
- */
300
- async function getFfmpegBin() {
301
- const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
302
- for (const p of candidates) {
303
- if (existsSync(p))
304
- return p;
305
- }
306
- return "ffmpeg";
307
- }
308
332
  /** Fixed filename mlx_audio.tts.generate writes its output clip as. */
309
333
  const TTS_OUTPUT_FILENAME = "audio_000.mp3";
310
334
  /**
@@ -574,6 +598,76 @@ export async function generateSceneVideo(uvPath, projectId, scene, characters, o
574
598
  url: `/api/files?path=${encodeURIComponent(videoPath)}`,
575
599
  };
576
600
  }
601
+ /**
602
+ * Generate a video from a project image via LTX-2.3. `imagePath` must be a bare
603
+ * filename previously uploaded/generated for this project. Used by the queue
604
+ * worker for the "Scene Video Generation" tab.
605
+ */
606
+ export async function generateImageToVideo(uvPath, projectId, params, onLog) {
607
+ const { prompt, imagePath, mode } = params;
608
+ if (!isValidProjectId(projectId))
609
+ return { error: "Invalid project ID" };
610
+ if (!prompt || !prompt.trim())
611
+ return { error: "Prompt is required" };
612
+ if (!imagePath)
613
+ return { error: "Image path is required" };
614
+ const resolvedImage = resolveSafePath(imagePath, projectId);
615
+ if (!resolvedImage) {
616
+ return {
617
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
618
+ };
619
+ }
620
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
621
+ if (!existsSync(ltxFolder)) {
622
+ return { error: "ltx-2-mlx not found. Run setup first." };
623
+ }
624
+ const projectOutputDir = resolveOutputDir(undefined, projectId);
625
+ if (!projectOutputDir)
626
+ return { error: "Invalid output directory." };
627
+ const outputFile = `video-${Date.now()}.mp4`;
628
+ const outputPath = join(projectOutputDir, outputFile);
629
+ const videoWidth = Number(params.width) || 480;
630
+ const videoHeight = Number(params.height) || 480;
631
+ const videoFrames = Number(params.frames) || 121;
632
+ const videoFps = Number(params.frameRate) || 24;
633
+ const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 30);
634
+ const stage2Steps = Math.max(1, Math.round(Number(params.stage2Steps)) || 3);
635
+ const result = await runCommand([
636
+ uvPath,
637
+ "run",
638
+ "ltx-2-mlx",
639
+ "generate",
640
+ "--model",
641
+ resolveLtxModel(params.model),
642
+ "--prompt",
643
+ prompt.trim(),
644
+ stageFlagFor(mode),
645
+ "--stage1-steps",
646
+ String(stage1Steps),
647
+ "--stage2-steps",
648
+ String(stage2Steps),
649
+ "--frames",
650
+ String(videoFrames),
651
+ "--width",
652
+ String(videoWidth),
653
+ "--height",
654
+ String(videoHeight),
655
+ "--frame-rate",
656
+ String(videoFps),
657
+ "--image",
658
+ resolvedImage,
659
+ "--output",
660
+ outputPath,
661
+ ], { cwd: ltxFolder, onLog });
662
+ if (!result.success || !existsSync(outputPath)) {
663
+ return { error: result.output || "Video generation failed" };
664
+ }
665
+ backupFile(outputPath, projectId);
666
+ return {
667
+ filename: outputFile,
668
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
669
+ };
670
+ }
577
671
  /** Normalize a value into a safe filesystem slug. */
578
672
  function slugify(v) {
579
673
  return String(v || "")
@@ -584,10 +678,11 @@ function slugify(v) {
584
678
  * Generate a single scene image via fast-image-edit (FLUX.2 Klein), using the
585
679
  * already-generated character/place images referenced by the scene's slugs.
586
680
  */
587
- export async function generateSceneImage(projectId, scene, onLog) {
681
+ export async function generateSceneImage(projectId, scene, steps, onLog) {
588
682
  const s = slugify(scene?.slug);
589
683
  if (!s)
590
684
  return { error: "Invalid scene slug" };
685
+ const stepCount = Math.max(1, Number(steps) || 6);
591
686
  const outputDir = join(OUTPUT_DIR, projectId);
592
687
  const mlxgen = await getMlxgenBin();
593
688
  const refImages = [];
@@ -611,7 +706,7 @@ export async function generateSceneImage(projectId, scene, onLog) {
611
706
  const fluxArgs = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
612
707
  for (const p of refImages)
613
708
  fluxArgs.push("--image", p);
614
- fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", "5", "--seed", "42", "--width", "1024", "--height", "1024");
709
+ fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
615
710
  const result = await runCommand(fluxArgs, { onLog });
616
711
  if (!result.success || !existsSync(sceneImagePath)) {
617
712
  return { error: result.output || `Failed to generate scene image ${s}` };
@@ -622,12 +717,353 @@ export async function generateSceneImage(projectId, scene, onLog) {
622
717
  url: `/api/files?path=${encodeURIComponent(sceneImagePath)}`,
623
718
  };
624
719
  }
720
+ /**
721
+ * Upscale/refine a project image via mlxgen (SeedVR2). `imagePath` must be a
722
+ * bare filename previously uploaded/generated for this project. `resolution` is
723
+ * either "1x" (refine at native resolution) or "2048" (upscale to 2048px).
724
+ */
725
+ export async function generateUpscale(projectId, imagePath, resolution, onLog) {
726
+ if (!isValidProjectId(projectId))
727
+ return { error: "Invalid project ID" };
728
+ if (!imagePath)
729
+ return { error: "Image path is required" };
730
+ const resolvedImage = resolveSafePath(imagePath, projectId);
731
+ if (!resolvedImage) {
732
+ return {
733
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
734
+ };
735
+ }
736
+ const target = /^(1x|\d+)$/.test(resolution) ? resolution : "1x";
737
+ const mlxgen = await getMlxgenBin();
738
+ const projectOutputDir = join(OUTPUT_DIR, projectId);
739
+ ensureDir(projectOutputDir);
740
+ const outputFile = `upscale-${target}-${Date.now()}.png`;
741
+ const outputPath = join(projectOutputDir, outputFile);
742
+ const result = await runCommand([
743
+ mlxgen,
744
+ "upscale",
745
+ "--model",
746
+ SEEDVR2_MODEL,
747
+ "--image-path",
748
+ resolvedImage,
749
+ "--resolution",
750
+ target,
751
+ "--seed",
752
+ "42",
753
+ "--mlx-cache-limit-gb",
754
+ "100",
755
+ "--output",
756
+ outputPath,
757
+ ], { onLog });
758
+ if (!result.success || !existsSync(outputPath)) {
759
+ return { error: result.output || "Upscale failed" };
760
+ }
761
+ backupFile(outputPath, projectId);
762
+ return {
763
+ filename: outputFile,
764
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
765
+ };
766
+ }
767
+ /**
768
+ * Clone a reference voice and speak `text` via mlx_audio.tts.generate. `refAudioPath`
769
+ * must be a bare filename previously uploaded to this project. `quality` is "low"
770
+ * or "high" (maps to a TTS model). Output is saved under <output>/voices/.
771
+ */
772
+ export async function generateVoiceClone(uvPath, projectId, text, refAudioPath, quality, onLog) {
773
+ if (!isValidProjectId(projectId))
774
+ return { error: "Invalid project ID" };
775
+ if (!refAudioPath)
776
+ return { error: "Reference audio is required" };
777
+ // Sanitize the transcript: strip control characters (incl. newlines) and
778
+ // collapse whitespace. spawn() runs with shell:false (array args), so shell
779
+ // metacharacters cannot execute, but this keeps `--text` a single well-formed
780
+ // argument and out of the terminal log.
781
+ const cleanText = text
782
+ .replace(/[\u0000-\u001f\u007f]/g, " ")
783
+ .replace(/\s+/g, " ")
784
+ .trim();
785
+ if (!cleanText)
786
+ return { error: "Text is required" };
787
+ const resolvedRef = resolveSafePath(refAudioPath, projectId);
788
+ if (!resolvedRef) {
789
+ return {
790
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
791
+ };
792
+ }
793
+ const model = quality === "low" ? TTS_MODELS.low : TTS_MODELS.high;
794
+ const projectOutputDir = resolveOutputDir(undefined, projectId);
795
+ if (!projectOutputDir)
796
+ return { error: "Invalid output directory." };
797
+ const voiceId = `voice-${Date.now()}`;
798
+ const voiceDir = join(projectOutputDir, "voices", voiceId);
799
+ ensureDir(voiceDir);
800
+ const result = await runCommand([
801
+ uvPath,
802
+ "run",
803
+ "mlx_audio.tts.generate",
804
+ "--model",
805
+ model,
806
+ "--text",
807
+ cleanText,
808
+ "--ref_audio",
809
+ resolvedRef,
810
+ "--output",
811
+ voiceDir,
812
+ "--audio_format",
813
+ "mp3",
814
+ "--play",
815
+ "--instruct",
816
+ "slow down speech",
817
+ ], { cwd: voiceDir, onLog });
818
+ if (!result.success) {
819
+ return { error: result.output || "Voice generation failed" };
820
+ }
821
+ const path = resolveAudioFile(voiceDir);
822
+ if (!path) {
823
+ return { error: "TTS completed but no audio file was produced" };
824
+ }
825
+ const filename = path.split(sep).pop() || TTS_OUTPUT_FILENAME;
826
+ // Persist a per-voice metadata file so the generated-voice list can be rebuilt
827
+ // from the folder structure (each voice in its own folder with meta.json), with
828
+ // no central JSON index.
829
+ const meta = {
830
+ id: voiceId,
831
+ transcript: cleanText,
832
+ quality,
833
+ refAudioFilename: refAudioPath,
834
+ filename,
835
+ createdAt: new Date().toISOString(),
836
+ };
837
+ writeFileSync(join(voiceDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
838
+ return {
839
+ filename,
840
+ url: `/api/files?path=${encodeURIComponent(path)}`,
841
+ };
842
+ }
843
+ /** Find the newest video file (mp4/webm/mov) inside a directory. */
844
+ function findNewestVideo(dir) {
845
+ let entries = [];
846
+ try {
847
+ entries = readdirSync(dir);
848
+ }
849
+ catch {
850
+ return null;
851
+ }
852
+ let newest = null;
853
+ for (const name of entries) {
854
+ const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
855
+ if (ext !== ".mp4" && ext !== ".webm" && ext !== ".mov")
856
+ continue;
857
+ const full = join(dir, name);
858
+ try {
859
+ const st = statSync(full);
860
+ if (!st.isFile())
861
+ continue;
862
+ if (!newest || st.mtimeMs > newest.mtime) {
863
+ newest = { path: full, mtime: st.mtimeMs };
864
+ }
865
+ }
866
+ catch {
867
+ // skip unreadable entries
868
+ }
869
+ }
870
+ return newest ? newest.path : null;
871
+ }
872
+ /**
873
+ * Generate a video from an image + audio via ltx-2-mlx `a2v` (audio-to-video).
874
+ * `imagePath` and `audioPath` must be bare filenames previously uploaded/generated
875
+ * for this project. `stage1Steps` maps to `--stage1-steps` (15 default, 30 HD).
876
+ */
877
+ export async function generateAudioToVideo(uvPath, projectId, params, onLog) {
878
+ if (!isValidProjectId(projectId))
879
+ return { error: "Invalid project ID" };
880
+ if (!params.imagePath)
881
+ return { error: "Image path is required" };
882
+ if (!params.audioPath)
883
+ return { error: "Audio path is required" };
884
+ const resolvedImage = resolveSafePath(params.imagePath, projectId);
885
+ if (!resolvedImage) {
886
+ return {
887
+ error: "Invalid image path. Provide a filename previously uploaded to this project.",
888
+ };
889
+ }
890
+ const resolvedAudio = resolveSafePath(params.audioPath, projectId);
891
+ if (!resolvedAudio) {
892
+ return {
893
+ error: "Invalid audio path. Provide a filename previously uploaded to this project.",
894
+ };
895
+ }
896
+ // Collapse whitespace so the prompt stays a single well-formed CLI argument.
897
+ const cleanPrompt = String(params.prompt || "")
898
+ .replace(/\s+/g, " ")
899
+ .trim() || "scene";
900
+ const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 15);
901
+ // 1 second = 24 frames, plus a terminal frame (24n + 1).
902
+ const frames = Math.max(1, Math.round(Number(params.frames)) || 25);
903
+ const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
904
+ if (!existsSync(ltxFolder)) {
905
+ return { error: "ltx-2-mlx not found. Run setup first." };
906
+ }
907
+ const outputDir = join(OUTPUT_DIR, projectId, `a2v-${Date.now()}`);
908
+ ensureDir(outputDir);
909
+ const result = await runCommand([
910
+ uvPath,
911
+ "run",
912
+ "ltx-2-mlx",
913
+ "a2v",
914
+ "--image",
915
+ resolvedImage,
916
+ "--audio",
917
+ resolvedAudio,
918
+ "--frame-rate",
919
+ "24",
920
+ "--frames",
921
+ String(frames),
922
+ "--output",
923
+ outputDir,
924
+ "--prompt",
925
+ cleanPrompt,
926
+ "--stage1-steps",
927
+ String(stage1Steps),
928
+ "--stage2-steps",
929
+ "3",
930
+ ], { cwd: ltxFolder, onLog });
931
+ if (!result.success) {
932
+ return { error: result.output || "Audio-to-video generation failed" };
933
+ }
934
+ const videoPath = findNewestVideo(outputDir);
935
+ if (!videoPath) {
936
+ return { error: "a2v completed but no video file was produced" };
937
+ }
938
+ backupFile(videoPath, projectId);
939
+ return {
940
+ filename: videoPath.split(sep).pop() || "a2v.mp4",
941
+ url: `/api/files?path=${encodeURIComponent(videoPath)}`,
942
+ };
943
+ }
944
+ /**
945
+ * Clone a reference voice via dots-tts. Output is written to
946
+ * <output>/<projectId>/dots-tts/ as `<prefix>_000.wav`.
947
+ */
948
+ export async function generateAdvancedVoiceClone(projectId, params, onLog) {
949
+ if (!isValidProjectId(projectId))
950
+ return { error: "Invalid project ID" };
951
+ const cleanText = String(params.text || "")
952
+ .replace(/\s+/g, " ")
953
+ .trim();
954
+ if (!cleanText)
955
+ return { error: "Text is required" };
956
+ if (!params.refAudioPath)
957
+ return { error: "Reference audio is required" };
958
+ const resolvedRef = resolveSafePath(params.refAudioPath, projectId);
959
+ if (!resolvedRef) {
960
+ return {
961
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
962
+ };
963
+ }
964
+ const language = String(params.language || "YUE").trim() || "YUE";
965
+ const prefix = String(params.outPrefix || "voice")
966
+ .replace(/[^a-zA-Z0-9_-]/g, "_")
967
+ .slice(0, 64) || "voice";
968
+ const model = resolveDotsTtsModel(String(params.model || ""));
969
+ // dots-tts expects a WAV reference — convert mp3/other formats to 16-bit PCM WAV.
970
+ let refAudio = resolvedRef;
971
+ if (!resolvedRef.toLowerCase().endsWith(".wav")) {
972
+ const tempDir = join(TEMP_DIR, String(projectId));
973
+ ensureDir(tempDir);
974
+ const convertedRef = join(tempDir, `avc-ref-${Date.now()}.wav`);
975
+ const ffmpegBin = await getFfmpegBin();
976
+ const conv = await runCommand([
977
+ ffmpegBin,
978
+ "-y",
979
+ "-i",
980
+ resolvedRef,
981
+ "-c:a",
982
+ "pcm_s16le",
983
+ "-ar",
984
+ "44100",
985
+ "-ac",
986
+ "2",
987
+ convertedRef,
988
+ ], { onLog });
989
+ if (!conv.success || !existsSync(convertedRef)) {
990
+ return { error: "Failed to convert reference audio to WAV" };
991
+ }
992
+ refAudio = convertedRef;
993
+ }
994
+ const dotsTtsBin = await getDotsTtsBin();
995
+ // Mirror the voice-clone tab's timestamped-folder layout so outputs never
996
+ // overwrite each other.
997
+ const voiceId = `voice-${Date.now()}`;
998
+ const outDir = join(OUTPUT_DIR, projectId, "dots-tts", voiceId);
999
+ ensureDir(outDir);
1000
+ console.log("debug-info:", [
1001
+ dotsTtsBin,
1002
+ "--model",
1003
+ model,
1004
+ "--text",
1005
+ cleanText || "please provide text",
1006
+ "--ref-audio",
1007
+ refAudio,
1008
+ "--language",
1009
+ language,
1010
+ "--out-path",
1011
+ outDir,
1012
+ "--out-prefix",
1013
+ prefix,
1014
+ "--max-generate-length",
1015
+ "3000",
1016
+ ]);
1017
+ const result = await runCommand([
1018
+ dotsTtsBin,
1019
+ "--model",
1020
+ model,
1021
+ "--text",
1022
+ cleanText || "please provide text",
1023
+ "--ref-audio",
1024
+ refAudio,
1025
+ "--language",
1026
+ language,
1027
+ "--out-path",
1028
+ outDir,
1029
+ "--out-prefix",
1030
+ prefix,
1031
+ "--max-generate-length",
1032
+ "3000",
1033
+ ], { cwd: dirname(dotsTtsBin), onLog });
1034
+ if (!result.success) {
1035
+ return { error: result.output || "Advanced voice clone failed" };
1036
+ }
1037
+ const outputFile = `${prefix}_000.wav`;
1038
+ const outputPath = join(outDir, outputFile);
1039
+ if (!existsSync(outputPath)) {
1040
+ return { error: `Expected output ${outputFile} was not produced` };
1041
+ }
1042
+ // Mirror the voice-clone tab's on-disk conventions: a timestamped backup copy
1043
+ // plus a per-voice meta.json so the output can be listed without a central index.
1044
+ backupFile(outputPath, projectId);
1045
+ const meta = {
1046
+ id: voiceId,
1047
+ transcript: cleanText,
1048
+ language,
1049
+ model,
1050
+ refAudioFilename: params.refAudioPath,
1051
+ outPrefix: prefix,
1052
+ filename: outputFile,
1053
+ createdAt: new Date().toISOString(),
1054
+ };
1055
+ writeFileSync(join(outDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
1056
+ return {
1057
+ filename: outputFile,
1058
+ url: `/api/files?path=${encodeURIComponent(outputPath)}`,
1059
+ };
1060
+ }
625
1061
  /**
626
1062
  * Generate a composite image via fast-image-edit (FLUX.2 Klein). `images` are
627
1063
  * base64 data URLs that are decoded into temp files and passed to the model as
628
1064
  * separate `--image` inputs. Used by the generation queue worker.
629
1065
  */
630
- export async function generateFastImageEditImage(projectId, prompt, images, onLog) {
1066
+ export async function generateFastImageEditImage(projectId, prompt, images, steps, upscaleResolution, onLog) {
631
1067
  if (!isValidProjectId(projectId))
632
1068
  return { error: "Invalid project ID" };
633
1069
  if (!prompt || !prompt.trim())
@@ -635,6 +1071,7 @@ export async function generateFastImageEditImage(projectId, prompt, images, onLo
635
1071
  if (!Array.isArray(images) || images.length === 0) {
636
1072
  return { error: "At least one reference image is required" };
637
1073
  }
1074
+ const stepCount = Math.max(1, Number(steps) || 4);
638
1075
  // Decode each base64 reference image into a temp workspace file so the
639
1076
  // FLUX model receives them as separate `--image` inputs.
640
1077
  const tempDir = join(TEMP_DIR, String(projectId));
@@ -661,12 +1098,21 @@ export async function generateFastImageEditImage(projectId, prompt, images, onLo
661
1098
  const args = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
662
1099
  for (const path of tempImagePaths)
663
1100
  args.push("--image", path);
664
- args.push("--prompt", prompt.trim(), "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps", "5", "--seed", "42", "--width", "1024", "--height", "1024");
1101
+ args.push("--prompt", prompt.trim(), "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
665
1102
  const result = await runCommand(args, { onLog });
666
1103
  if (!result.success || !existsSync(outputPath)) {
667
1104
  return { error: result.output || "Fast image edit failed" };
668
1105
  }
669
1106
  backupFile(outputPath, projectId);
1107
+ // Optionally upscale the generated result (1x / 1500px / 2000px).
1108
+ if (upscaleResolution && upscaleResolution !== "none") {
1109
+ if (onLog)
1110
+ onLog(`Upscaling result (${upscaleResolution})…\n`);
1111
+ const upscaled = await generateUpscale(projectId, outputFile, upscaleResolution, onLog);
1112
+ if ("error" in upscaled)
1113
+ return { error: upscaled.error };
1114
+ return upscaled;
1115
+ }
670
1116
  return {
671
1117
  filename: outputFile,
672
1118
  url: `/api/files?path=${encodeURIComponent(outputPath)}`,
@@ -973,6 +1419,60 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
973
1419
  raw.sort((a, b) => b.birthtime - a.birthtime);
974
1420
  res.json(raw.map(({ filename, url }) => ({ filename, url })));
975
1421
  });
1422
+ // List previously generated voice clones by scanning the voices folder. Each
1423
+ // voice lives in its own subfolder with a meta.json (no central JSON index).
1424
+ app.get("/api/projects/:id/voices", (req, res) => {
1425
+ const { id } = req.params;
1426
+ if (!isValidProjectId(id)) {
1427
+ res.status(400).json({ error: "Invalid project ID" });
1428
+ return;
1429
+ }
1430
+ const voicesRoot = join(OUTPUT_DIR, id, "voices");
1431
+ const results = [];
1432
+ if (existsSync(voicesRoot)) {
1433
+ let names = [];
1434
+ try {
1435
+ names = readdirSync(voicesRoot);
1436
+ }
1437
+ catch {
1438
+ names = [];
1439
+ }
1440
+ for (const name of names) {
1441
+ const voiceDir = join(voicesRoot, name);
1442
+ let isDir = false;
1443
+ try {
1444
+ isDir = statSync(voiceDir).isDirectory();
1445
+ }
1446
+ catch {
1447
+ isDir = false;
1448
+ }
1449
+ if (!isDir)
1450
+ continue;
1451
+ let meta = null;
1452
+ try {
1453
+ meta = JSON.parse(readFileSync(join(voiceDir, "meta.json"), "utf-8"));
1454
+ }
1455
+ catch {
1456
+ continue; // folder without a meta.json — skip
1457
+ }
1458
+ const filename = String(meta?.filename || TTS_OUTPUT_FILENAME);
1459
+ const audioPath = join(voiceDir, filename);
1460
+ if (!existsSync(audioPath))
1461
+ continue;
1462
+ results.push({
1463
+ id: name,
1464
+ transcript: String(meta?.transcript ?? ""),
1465
+ quality: String(meta?.quality ?? "high"),
1466
+ refAudioFilename: meta?.refAudioFilename ?? null,
1467
+ filename,
1468
+ createdAt: meta?.createdAt ?? null,
1469
+ url: `/api/files?path=${encodeURIComponent(audioPath)}`,
1470
+ });
1471
+ }
1472
+ }
1473
+ results.sort((a, b) => String(b.createdAt ?? "").localeCompare(String(a.createdAt ?? "")));
1474
+ res.json(results);
1475
+ });
976
1476
  // ========== Render: Text-to-Image ==========
977
1477
  app.post("/api/render/text-to-image", async (req, res) => {
978
1478
  const { prompt, projectId, width = 512, height = 512, device = "mps", } = req.body || {};
@@ -1066,7 +1566,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1066
1566
  });
1067
1567
  // ========== Render: Image-to-Video ==========
1068
1568
  app.post("/api/render/image-to-video", async (req, res) => {
1069
- const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", } = req.body || {};
1569
+ const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", stage1Steps, stage2Steps, model, } = req.body || {};
1070
1570
  if (!prompt) {
1071
1571
  res.status(400).json({ error: "Prompt is required" });
1072
1572
  return;
@@ -1122,6 +1622,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1122
1622
  const videoHeight = Number(height) || 480;
1123
1623
  const videoFrames = Number(frames) || 121;
1124
1624
  const videoFps = Number(frameRate) || 24;
1625
+ const stage1 = Math.max(1, Math.round(Number(stage1Steps)) || 30);
1626
+ const stage2 = Math.max(1, Math.round(Number(stage2Steps)) || 3);
1125
1627
  send("progress", {
1126
1628
  status: "starting",
1127
1629
  label: "Generating video...",
@@ -1139,10 +1641,14 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1139
1641
  "ltx-2-mlx",
1140
1642
  "generate",
1141
1643
  "--model",
1142
- "dgrauet/ltx-2.3-mlx-q8",
1644
+ resolveLtxModel(model),
1143
1645
  "--prompt",
1144
1646
  prompt,
1145
1647
  stageFlagFor(mode),
1648
+ "--stage1-steps",
1649
+ String(stage1),
1650
+ "--stage2-steps",
1651
+ String(stage2),
1146
1652
  // "--low-ram",
1147
1653
  "--frames",
1148
1654
  String(videoFrames),
@@ -1738,26 +2244,39 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1738
2244
  res.end();
1739
2245
  }
1740
2246
  });
1741
- // ========== Render: Extend-Video ==========
1742
- app.post("/api/render/extend-video", async (req, res) => {
1743
- const { prompt, videoPath, projectId, extendFrames = 2 } = req.body || {};
1744
- if (!prompt) {
1745
- res.status(400).json({ error: "Prompt is required" });
2247
+ // ========== MLX-Audio: Status ==========
2248
+ app.get("/api/mlxaudio/status", async (_req, res) => {
2249
+ // Never throw on a missing uv installation state is based on the folder.
2250
+ try {
2251
+ await getUvPath();
2252
+ }
2253
+ catch {
2254
+ // uv not found; installed is still reported from the folder below.
2255
+ }
2256
+ res.json({
2257
+ installed: existsSync(join(PYTHON_DIR, "mlx-audio")),
2258
+ });
2259
+ });
2260
+ // ========== Render: Voice Chat (TTS with reference voice) ==========
2261
+ app.post("/api/render/voice-chat", async (req, res) => {
2262
+ const { text, refAudioPath, projectId } = req.body || {};
2263
+ if (!text || typeof text !== "string" || !text.trim()) {
2264
+ res.status(400).json({ error: "Text is required" });
1746
2265
  return;
1747
2266
  }
1748
- if (!videoPath) {
1749
- res.status(400).json({ error: "Video path is required" });
2267
+ if (!refAudioPath) {
2268
+ res.status(400).json({ error: "Reference audio is required" });
1750
2269
  return;
1751
2270
  }
1752
- if (!projectId) {
1753
- res.status(400).json({ error: "Project ID is required" });
2271
+ if (!projectId || !isValidProjectId(String(projectId))) {
2272
+ res.status(400).json({ error: "Invalid project ID" });
1754
2273
  return;
1755
2274
  }
1756
- // Resolve video path — only bare .mp4 filenames in this project's output dir
1757
- const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
1758
- if (!resolvedVideo) {
2275
+ // Resolve reference audio — only bare filenames in this project's dirs.
2276
+ const resolvedRef = resolveSafePath(refAudioPath, String(projectId));
2277
+ if (!resolvedRef) {
1759
2278
  res.status(400).json({
1760
- error: "Invalid video path. Provide a filename previously generated in this project.",
2279
+ error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
1761
2280
  });
1762
2281
  return;
1763
2282
  }
@@ -1772,59 +2291,65 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1772
2291
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
1773
2292
  };
1774
2293
  try {
1775
- const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
1776
- if (!existsSync(ltxFolder)) {
1777
- send("error", { error: "ltx-2-mlx not found. Run setup first." });
2294
+ const uvPath = await getUvPath();
2295
+ const projectOutputDir = resolveOutputDir(null, String(projectId));
2296
+ if (!projectOutputDir) {
2297
+ send("error", { error: "Invalid output directory." });
1778
2298
  res.end();
1779
2299
  return;
1780
2300
  }
1781
- const uvPath = await getUvPath();
1782
- const projectOutputDir = join(OUTPUT_DIR, projectId);
1783
- ensureDir(projectOutputDir);
1784
- const outputFile = `extended-${Date.now()}.mp4`;
1785
- const outputPath = join(projectOutputDir, outputFile);
1786
- const framesToAdd = Number(extendFrames) || 2;
2301
+ // Isolate each utterance so a new response never overwrites a previous
2302
+ // one. mlx_audio writes the clip as <dir>/audio_000.mp3.
2303
+ const voiceDir = join(projectOutputDir, "voice-chat", `chat-${Date.now()}`);
2304
+ ensureDir(voiceDir);
1787
2305
  send("progress", {
1788
2306
  status: "starting",
1789
- label: "Extending video...",
1790
- inputFile: resolvedVideo,
1791
- outputFile,
1792
- settings: { extendFrames: framesToAdd },
2307
+ label: "Generating voice...",
1793
2308
  });
1794
2309
  const proc = spawn([
1795
2310
  uvPath,
1796
2311
  "run",
1797
- "ltx-2-mlx",
1798
- "extend",
2312
+ "mlx_audio.tts.generate",
1799
2313
  "--model",
1800
- "dgrauet/ltx-2.3-mlx-q8",
1801
- "--prompt",
1802
- prompt,
1803
- "--video",
1804
- resolvedVideo,
1805
- "--extend-frames",
1806
- String(framesToAdd),
2314
+ TTS_MODELS.high,
2315
+ "--text",
2316
+ text.trim(),
2317
+ "--ref_audio",
2318
+ resolvedRef,
2319
+ // "--play",
1807
2320
  "--output",
1808
- outputPath,
2321
+ voiceDir,
2322
+ "--audio_format",
2323
+ "mp3",
2324
+ // "--stream",
2325
+ // "--save",
2326
+ "--instruct",
2327
+ "slow down",
1809
2328
  ], {
1810
2329
  env: process.env,
1811
- cwd: ltxFolder,
2330
+ cwd: voiceDir,
1812
2331
  stdout: "pipe",
1813
2332
  stderr: "pipe",
1814
2333
  });
1815
2334
  activeProc = proc;
1816
- // Stream stdout/stderr concurrently
1817
- const stdoutPromise = streamToSSE(proc.stdout, "Extend", send);
1818
- const stderrText = await streamToSSE(proc.stderr, "Extend", send);
2335
+ const stdoutPromise = streamToSSE(proc.stdout, "VoiceChat", send);
2336
+ const stderrText = await streamToSSE(proc.stderr, "VoiceChat", send);
1819
2337
  await stdoutPromise;
1820
2338
  const exitCode = await proc.exited;
1821
- const success = exitCode === 0 && existsSync(outputPath);
1822
- if (success) {
1823
- send("complete", {
1824
- success: true,
1825
- path: outputPath,
1826
- filename: outputFile,
1827
- });
2339
+ if (exitCode === 0) {
2340
+ const path = resolveAudioFile(voiceDir);
2341
+ if (path) {
2342
+ send("complete", {
2343
+ success: true,
2344
+ path,
2345
+ filename: path.split(sep).pop(),
2346
+ });
2347
+ }
2348
+ else {
2349
+ send("error", {
2350
+ error: "TTS completed but no audio file was produced",
2351
+ });
2352
+ }
1828
2353
  }
1829
2354
  else {
1830
2355
  send("error", {
@@ -1841,47 +2366,17 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1841
2366
  res.end();
1842
2367
  }
1843
2368
  });
1844
- // ========== MLX-Audio: Status ==========
1845
- app.get("/api/mlxaudio/status", async (_req, res) => {
1846
- // Never throw on a missing uv — installation state is based on the folder.
1847
- try {
1848
- await getUvPath();
1849
- }
1850
- catch {
1851
- // uv not found; installed is still reported from the folder below.
1852
- }
2369
+ // ========== MLX-Gen: Status ==========
2370
+ app.get("/api/mlxgen/status", (_req, res) => {
1853
2371
  res.json({
1854
- installed: existsSync(join(PYTHON_DIR, "mlx-audio")),
2372
+ installed: isMlxgenInstalled(),
2373
+ zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
2374
+ fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
2375
+ seedvr2Downloaded: isModelDownloaded(SEEDVR2_MODEL),
1855
2376
  });
1856
2377
  });
1857
- // ========== Render: Text-to-Speech (mlx-audio) ==========
1858
- app.post("/api/render/tts", async (req, res) => {
1859
- const { text, refAudioPath, projectId, outputDir, quality, voiceId } = req.body || {};
1860
- if (!text || typeof text !== "string" || !text.trim()) {
1861
- res.status(400).json({ error: "Text is required" });
1862
- return;
1863
- }
1864
- if (!refAudioPath) {
1865
- res.status(400).json({ error: "Reference audio is required" });
1866
- return;
1867
- }
1868
- if (!projectId || !isValidProjectId(String(projectId))) {
1869
- res.status(400).json({ error: "Invalid project ID" });
1870
- return;
1871
- }
1872
- if (quality !== "low" && quality !== "high") {
1873
- res.status(400).json({ error: "Quality must be 'low' or 'high'" });
1874
- return;
1875
- }
1876
- // Resolve reference audio — only bare filenames in this project's dirs
1877
- const resolvedRef = resolveSafePath(refAudioPath, projectId);
1878
- if (!resolvedRef) {
1879
- res.status(400).json({
1880
- error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
1881
- });
1882
- return;
1883
- }
1884
- // SSE headers
2378
+ // ========== MLX-Gen: Install ==========
2379
+ app.post("/api/mlxgen/install", async (_req, res) => {
1885
2380
  res.writeHead(200, {
1886
2381
  "Content-Type": "text/event-stream",
1887
2382
  "Cache-Control": "no-cache",
@@ -1893,81 +2388,21 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1893
2388
  };
1894
2389
  try {
1895
2390
  const uvPath = await getUvPath();
1896
- const projectOutputDir = resolveOutputDir(outputDir, projectId);
1897
- if (!projectOutputDir) {
1898
- send("error", { error: "Invalid output directory." });
1899
- res.end();
1900
- return;
1901
- }
1902
- // TTS audio is saved under <projectOutputDir>/voices/<voiceId>/ so each
1903
- // row's voiceover stays isolated in its own folder. Sanitize voiceId to
1904
- // alphanumerics/_/- so it can never escape the voices directory.
1905
- const safeVoiceId = String(voiceId ?? `voice-${Date.now()}`)
1906
- .replace(/[^a-zA-Z0-9_-]/g, "_")
1907
- .slice(0, 64);
1908
- const voiceDir = join(projectOutputDir, "voices", safeVoiceId);
1909
- ensureDir(voiceDir);
1910
- // Regenerating a row must overwrite the previous clip: delete any
1911
- // existing output so the generator writes a fresh audio_000.mp3.
1912
- const prevAudio = join(voiceDir, TTS_OUTPUT_FILENAME);
1913
- if (existsSync(prevAudio)) {
1914
- try {
1915
- unlinkSync(prevAudio);
1916
- }
1917
- catch {
1918
- // Ignore — the generator will overwrite the file regardless.
1919
- }
1920
- }
1921
2391
  send("progress", {
1922
2392
  status: "starting",
1923
- label: "Generating speech...",
1924
- model: TTS_MODELS[quality],
2393
+ label: "Installing mlx-gen...",
1925
2394
  });
1926
- // No --play flag: this is a silent server-side batch generation. The mp3
1927
- // lands in <projectOutputDir>/voices/<voiceId>/, which is servable via
1928
- // /api/files and resolvable via resolveSafePath for the mux step.
1929
- const proc = spawn([
1930
- uvPath,
1931
- "run",
1932
- "mlx_audio.tts.generate",
1933
- "--model",
1934
- TTS_MODELS[quality],
1935
- "--text",
1936
- text,
1937
- "--ref_audio",
1938
- resolvedRef,
1939
- "--output",
1940
- voiceDir,
1941
- "--audio_format",
1942
- "mp3",
1943
- "--play",
1944
- "--instruct",
1945
- "slow down speech",
1946
- ], {
1947
- env: process.env,
1948
- cwd: voiceDir,
2395
+ const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
1949
2396
  stdout: "pipe",
1950
2397
  stderr: "pipe",
1951
2398
  });
1952
2399
  activeProc = proc;
1953
- const stdoutPromise = streamToSSE(proc.stdout, "TTS", send);
1954
- const stderrText = await streamToSSE(proc.stderr, "TTS", send);
2400
+ const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
2401
+ const stderrText = await streamToSSE(proc.stderr, "Install", send);
1955
2402
  await stdoutPromise;
1956
2403
  const exitCode = await proc.exited;
1957
2404
  if (exitCode === 0) {
1958
- const path = resolveAudioFile(voiceDir);
1959
- if (path) {
1960
- send("complete", {
1961
- success: true,
1962
- path,
1963
- filename: path.split(sep).pop(),
1964
- });
1965
- }
1966
- else {
1967
- send("error", {
1968
- error: "TTS completed but no audio file was produced",
1969
- });
1970
- }
2405
+ send("complete", { success: true });
1971
2406
  }
1972
2407
  else {
1973
2408
  send("error", {
@@ -1984,30 +2419,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
1984
2419
  res.end();
1985
2420
  }
1986
2421
  });
1987
- // ========== Render: Voice Chat (TTS with reference voice) ==========
1988
- app.post("/api/render/voice-chat", async (req, res) => {
1989
- const { text, refAudioPath, projectId } = req.body || {};
1990
- if (!text || typeof text !== "string" || !text.trim()) {
1991
- res.status(400).json({ error: "Text is required" });
1992
- return;
1993
- }
1994
- if (!refAudioPath) {
1995
- res.status(400).json({ error: "Reference audio is required" });
1996
- return;
1997
- }
1998
- if (!projectId || !isValidProjectId(String(projectId))) {
1999
- res.status(400).json({ error: "Invalid project ID" });
2000
- return;
2001
- }
2002
- // Resolve reference audio — only bare filenames in this project's dirs.
2003
- const resolvedRef = resolveSafePath(refAudioPath, String(projectId));
2004
- if (!resolvedRef) {
2005
- res.status(400).json({
2006
- error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
2007
- });
2008
- return;
2009
- }
2010
- // SSE headers
2422
+ // ========== MLX-Gen: Download Z-Image Model ==========
2423
+ app.post("/api/mlxgen/download-z-model", async (_req, res) => {
2011
2424
  res.writeHead(200, {
2012
2425
  "Content-Type": "text/event-stream",
2013
2426
  "Cache-Control": "no-cache",
@@ -2017,66 +2430,24 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2017
2430
  const send = (event, data) => {
2018
2431
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2019
2432
  };
2433
+ const model = Z_IMAGE_MODEL;
2020
2434
  try {
2021
- const uvPath = await getUvPath();
2022
- const projectOutputDir = resolveOutputDir(null, String(projectId));
2023
- if (!projectOutputDir) {
2024
- send("error", { error: "Invalid output directory." });
2025
- res.end();
2026
- return;
2027
- }
2028
- // Isolate each utterance so a new response never overwrites a previous
2029
- // one. mlx_audio writes the clip as <dir>/audio_000.mp3.
2030
- const voiceDir = join(projectOutputDir, "voice-chat", `chat-${Date.now()}`);
2031
- ensureDir(voiceDir);
2435
+ const mlxgen = await getMlxgenBin();
2032
2436
  send("progress", {
2033
2437
  status: "starting",
2034
- label: "Generating voice...",
2438
+ label: `Downloading model ${model}...`,
2035
2439
  });
2036
- const proc = spawn([
2037
- uvPath,
2038
- "run",
2039
- "mlx_audio.tts.generate",
2040
- "--model",
2041
- TTS_MODELS.high,
2042
- "--text",
2043
- text.trim(),
2044
- "--ref_audio",
2045
- resolvedRef,
2046
- // "--play",
2047
- "--output",
2048
- voiceDir,
2049
- "--audio_format",
2050
- "mp3",
2051
- // "--stream",
2052
- // "--save",
2053
- "--instruct",
2054
- "slow down",
2055
- ], {
2056
- env: process.env,
2057
- cwd: voiceDir,
2440
+ const proc = spawn([mlxgen, "download", "--model", model], {
2058
2441
  stdout: "pipe",
2059
2442
  stderr: "pipe",
2060
2443
  });
2061
2444
  activeProc = proc;
2062
- const stdoutPromise = streamToSSE(proc.stdout, "VoiceChat", send);
2063
- const stderrText = await streamToSSE(proc.stderr, "VoiceChat", send);
2445
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2446
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2064
2447
  await stdoutPromise;
2065
2448
  const exitCode = await proc.exited;
2066
2449
  if (exitCode === 0) {
2067
- const path = resolveAudioFile(voiceDir);
2068
- if (path) {
2069
- send("complete", {
2070
- success: true,
2071
- path,
2072
- filename: path.split(sep).pop(),
2073
- });
2074
- }
2075
- else {
2076
- send("error", {
2077
- error: "TTS completed but no audio file was produced",
2078
- });
2079
- }
2450
+ send("complete", { success: true });
2080
2451
  }
2081
2452
  else {
2082
2453
  send("error", {
@@ -2093,38 +2464,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2093
2464
  res.end();
2094
2465
  }
2095
2466
  });
2096
- // ========== Render: Mux Video + Audio ==========
2097
- app.post("/api/render/mux-audio", async (req, res) => {
2098
- const { videoPath, audioPath, projectId, outputDir } = req.body || {};
2099
- if (!videoPath) {
2100
- res.status(400).json({ error: "Video path is required" });
2101
- return;
2102
- }
2103
- if (!audioPath) {
2104
- res.status(400).json({ error: "Audio path is required" });
2105
- return;
2106
- }
2107
- if (!projectId || !isValidProjectId(String(projectId))) {
2108
- res.status(400).json({ error: "Invalid project ID" });
2109
- return;
2110
- }
2111
- // Resolve video path — only bare .mp4 filenames in this project's output dir
2112
- const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
2113
- if (!resolvedVideo) {
2114
- res.status(400).json({
2115
- error: "Invalid video path. Provide a filename previously generated in this project.",
2116
- });
2117
- return;
2118
- }
2119
- // Resolve audio path — only bare filenames in this project's dirs
2120
- const resolvedAudio = resolveSafePath(audioPath, projectId);
2121
- if (!resolvedAudio) {
2122
- res.status(400).json({
2123
- error: "Invalid audio path. Provide a filename previously generated or uploaded to this project.",
2124
- });
2125
- return;
2126
- }
2127
- // SSE headers
2467
+ // ========== MLX-Gen: Download FLUX.2 Klein Model ==========
2468
+ app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
2128
2469
  res.writeHead(200, {
2129
2470
  "Content-Type": "text/event-stream",
2130
2471
  "Cache-Control": "no-cache",
@@ -2135,54 +2476,22 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2135
2476
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2136
2477
  };
2137
2478
  try {
2138
- const ffmpegBin = await getFfmpegBin();
2139
- const projectOutputDir = resolveOutputDir(outputDir, projectId);
2140
- if (!projectOutputDir) {
2141
- send("error", { error: "Invalid output directory." });
2142
- res.end();
2143
- return;
2144
- }
2145
- const finalFile = `voice-${Date.now()}.mp4`;
2146
- const finalPath = join(projectOutputDir, finalFile);
2479
+ const mlxgen = await getMlxgenBin();
2147
2480
  send("progress", {
2148
2481
  status: "starting",
2149
- label: "Muxing video and audio...",
2150
- outputFile: finalFile,
2482
+ label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
2151
2483
  });
2152
- const proc = spawn([
2153
- ffmpegBin,
2154
- "-y",
2155
- "-i",
2156
- resolvedVideo,
2157
- "-i",
2158
- resolvedAudio,
2159
- "-map",
2160
- "0:v",
2161
- "-map",
2162
- "1:a",
2163
- "-c:v",
2164
- "copy",
2165
- "-c:a",
2166
- "aac",
2167
- "-b:a",
2168
- "192k",
2169
- "-shortest",
2170
- finalPath,
2171
- ], {
2484
+ const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
2172
2485
  stdout: "pipe",
2173
2486
  stderr: "pipe",
2174
2487
  });
2175
2488
  activeProc = proc;
2176
- const stdoutPromise = streamToSSE(proc.stdout, "Mux", send);
2177
- const stderrText = await streamToSSE(proc.stderr, "Mux", send);
2489
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2490
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2178
2491
  await stdoutPromise;
2179
2492
  const exitCode = await proc.exited;
2180
- if (exitCode === 0 && existsSync(finalPath)) {
2181
- send("complete", {
2182
- success: true,
2183
- path: finalPath,
2184
- filename: finalFile,
2185
- });
2493
+ if (exitCode === 0) {
2494
+ send("complete", { success: true });
2186
2495
  }
2187
2496
  else {
2188
2497
  send("error", {
@@ -2199,16 +2508,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2199
2508
  res.end();
2200
2509
  }
2201
2510
  });
2202
- // ========== MLX-Gen: Status ==========
2203
- app.get("/api/mlxgen/status", (_req, res) => {
2204
- res.json({
2205
- installed: isMlxgenInstalled(),
2206
- zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
2207
- fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
2208
- });
2209
- });
2210
- // ========== MLX-Gen: Install ==========
2211
- app.post("/api/mlxgen/install", async (_req, res) => {
2511
+ // ========== MLX-Gen: Download SeedVR2 Model ==========
2512
+ app.post("/api/mlxgen/download-seedvr2-model", async (_req, res) => {
2212
2513
  res.writeHead(200, {
2213
2514
  "Content-Type": "text/event-stream",
2214
2515
  "Cache-Control": "no-cache",
@@ -2219,18 +2520,18 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2219
2520
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2220
2521
  };
2221
2522
  try {
2222
- const uvPath = await getUvPath();
2523
+ const mlxgen = await getMlxgenBin();
2223
2524
  send("progress", {
2224
2525
  status: "starting",
2225
- label: "Installing mlx-gen...",
2526
+ label: `Downloading model ${SEEDVR2_MODEL}...`,
2226
2527
  });
2227
- const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
2528
+ const proc = spawn([mlxgen, "download", "--model", SEEDVR2_MODEL], {
2228
2529
  stdout: "pipe",
2229
2530
  stderr: "pipe",
2230
2531
  });
2231
2532
  activeProc = proc;
2232
- const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
2233
- const stderrText = await streamToSSE(proc.stderr, "Install", send);
2533
+ const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2534
+ const stderrText = await streamToSSE(proc.stderr, "Download", send);
2234
2535
  await stdoutPromise;
2235
2536
  const exitCode = await proc.exited;
2236
2537
  if (exitCode === 0) {
@@ -2251,8 +2552,25 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2251
2552
  res.end();
2252
2553
  }
2253
2554
  });
2254
- // ========== MLX-Gen: Download Z-Image Model ==========
2255
- app.post("/api/mlxgen/download-z-model", async (_req, res) => {
2555
+ // ========== H3: Download Model ==========
2556
+ app.get("/api/h3/status", (_req, res) => {
2557
+ res.json({ downloaded: isH3ModelDownloaded() });
2558
+ });
2559
+ // ========== Hugging Face CLI + LTX Video Model ==========
2560
+ app.get("/api/hf/status", (_req, res) => {
2561
+ res.json({
2562
+ installed: whichSync("hf") !== null,
2563
+ ltxDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx-q8"),
2564
+ ltxBaseDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx"),
2565
+ ttsDownloaded: isModelDownloaded("Qwen/Qwen3-TTS-12Hz-1.7B-Base"),
2566
+ mlxVlmDownloaded: isModelDownloaded(MLX_VLM_MODEL),
2567
+ });
2568
+ });
2569
+ // ========== Dots-TTS: Status + Model Download ==========
2570
+ app.get("/api/dots-tts/status", (_req, res) => {
2571
+ res.json({ downloaded: isDotsTtsModelDownloaded() });
2572
+ });
2573
+ app.post("/api/dots-tts/download-model", async (_req, res) => {
2256
2574
  res.writeHead(200, {
2257
2575
  "Content-Type": "text/event-stream",
2258
2576
  "Cache-Control": "no-cache",
@@ -2262,20 +2580,24 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2262
2580
  const send = (event, data) => {
2263
2581
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2264
2582
  };
2265
- const model = Z_IMAGE_MODEL;
2266
2583
  try {
2267
- const mlxgen = await getMlxgenBin();
2584
+ ensureDir(DOTS_TTS_FOLDER);
2268
2585
  send("progress", {
2269
2586
  status: "starting",
2270
- label: `Downloading model ${model}...`,
2271
- });
2272
- const proc = spawn([mlxgen, "download", "--model", model], {
2273
- stdout: "pipe",
2274
- stderr: "pipe",
2587
+ label: "Downloading dots-tts model (int4)...",
2275
2588
  });
2589
+ const proc = spawn([
2590
+ "hf",
2591
+ "download",
2592
+ "shraey/dots-tts-mlx",
2593
+ "--include",
2594
+ "int4/*",
2595
+ "--local-dir",
2596
+ "./dots-tts-mlx-weights",
2597
+ ], { cwd: DOTS_TTS_FOLDER, stdout: "pipe", stderr: "pipe" });
2276
2598
  activeProc = proc;
2277
- const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2278
- const stderrText = await streamToSSE(proc.stderr, "Download", send);
2599
+ const stdoutPromise = streamToSSE(proc.stdout, "dots-tts", send);
2600
+ const stderrText = await streamToSSE(proc.stderr, "dots-tts", send);
2279
2601
  await stdoutPromise;
2280
2602
  const exitCode = await proc.exited;
2281
2603
  if (exitCode === 0) {
@@ -2296,8 +2618,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2296
2618
  res.end();
2297
2619
  }
2298
2620
  });
2299
- // ========== MLX-Gen: Download FLUX.2 Klein Model ==========
2300
- app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
2621
+ app.post("/api/hf/install", async (_req, res) => {
2301
2622
  res.writeHead(200, {
2302
2623
  "Content-Type": "text/event-stream",
2303
2624
  "Cache-Control": "no-cache",
@@ -2308,18 +2629,14 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2308
2629
  res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
2309
2630
  };
2310
2631
  try {
2311
- const mlxgen = await getMlxgenBin();
2312
2632
  send("progress", {
2313
2633
  status: "starting",
2314
- label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
2315
- });
2316
- const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
2317
- stdout: "pipe",
2318
- stderr: "pipe",
2634
+ label: "Installing huggingface-cli...",
2319
2635
  });
2636
+ const proc = spawn(["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash"], { stdout: "pipe", stderr: "pipe" });
2320
2637
  activeProc = proc;
2321
- const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
2322
- const stderrText = await streamToSSE(proc.stderr, "Download", send);
2638
+ const stdoutPromise = streamToSSE(proc.stdout, "HF Install", send);
2639
+ const stderrText = await streamToSSE(proc.stderr, "HF Install", send);
2323
2640
  await stdoutPromise;
2324
2641
  const exitCode = await proc.exited;
2325
2642
  if (exitCode === 0) {
@@ -2340,20 +2657,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2340
2657
  res.end();
2341
2658
  }
2342
2659
  });
2343
- // ========== H3: Download Model ==========
2344
- app.get("/api/h3/status", (_req, res) => {
2345
- res.json({ downloaded: isH3ModelDownloaded() });
2346
- });
2347
- // ========== Hugging Face CLI + LTX Video Model ==========
2348
- app.get("/api/hf/status", (_req, res) => {
2349
- res.json({
2350
- installed: whichSync("hf") !== null,
2351
- ltxDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx-q8"),
2352
- ttsDownloaded: isModelDownloaded("Qwen/Qwen3-TTS-12Hz-1.7B-Base"),
2353
- mlxVlmDownloaded: isModelDownloaded(MLX_VLM_MODEL),
2354
- });
2355
- });
2356
- app.post("/api/hf/install", async (_req, res) => {
2660
+ app.post("/api/hf/download-ltx", async (_req, res) => {
2357
2661
  res.writeHead(200, {
2358
2662
  "Content-Type": "text/event-stream",
2359
2663
  "Cache-Control": "no-cache",
@@ -2366,12 +2670,15 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2366
2670
  try {
2367
2671
  send("progress", {
2368
2672
  status: "starting",
2369
- label: "Installing huggingface-cli...",
2673
+ label: "Downloading dgrauet/ltx-2.3-mlx-q8...",
2674
+ });
2675
+ const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx-q8"], {
2676
+ stdout: "pipe",
2677
+ stderr: "pipe",
2370
2678
  });
2371
- const proc = spawn(["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash"], { stdout: "pipe", stderr: "pipe" });
2372
2679
  activeProc = proc;
2373
- const stdoutPromise = streamToSSE(proc.stdout, "HF Install", send);
2374
- const stderrText = await streamToSSE(proc.stderr, "HF Install", send);
2680
+ const stdoutPromise = streamToSSE(proc.stdout, "HF Download", send);
2681
+ const stderrText = await streamToSSE(proc.stderr, "HF Download", send);
2375
2682
  await stdoutPromise;
2376
2683
  const exitCode = await proc.exited;
2377
2684
  if (exitCode === 0) {
@@ -2392,7 +2699,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2392
2699
  res.end();
2393
2700
  }
2394
2701
  });
2395
- app.post("/api/hf/download-ltx", async (_req, res) => {
2702
+ app.post("/api/hf/download-ltx-base", async (_req, res) => {
2396
2703
  res.writeHead(200, {
2397
2704
  "Content-Type": "text/event-stream",
2398
2705
  "Cache-Control": "no-cache",
@@ -2405,9 +2712,9 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
2405
2712
  try {
2406
2713
  send("progress", {
2407
2714
  status: "starting",
2408
- label: "Downloading dgrauet/ltx-2.3-mlx-q8...",
2715
+ label: "Downloading dgrauet/ltx-2.3-mlx...",
2409
2716
  });
2410
- const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx-q8"], {
2717
+ const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx"], {
2411
2718
  stdout: "pipe",
2412
2719
  stderr: "pipe",
2413
2720
  });
@@ -3041,37 +3348,6 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
3041
3348
  writeCharacters(characters);
3042
3349
  res.json(removed);
3043
3350
  });
3044
- // Save an extracted video frame into the project's extracted-frames folder.
3045
- app.post("/api/extracted-frames", (req, res) => {
3046
- const { image, filename, projectId } = req.body || {};
3047
- if (!image) {
3048
- res.status(400).json({ error: "Image data is required (base64)" });
3049
- return;
3050
- }
3051
- if (!projectId || !isValidProjectId(String(projectId))) {
3052
- res.status(400).json({ error: "Invalid project ID" });
3053
- return;
3054
- }
3055
- try {
3056
- const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
3057
- const buffer = Buffer.from(base64, "base64");
3058
- const dir = join(EXTRACTED_FRAMES_DIR, String(projectId));
3059
- ensureDir(dir);
3060
- const safeName = (filename || `frame-${Date.now()}.png`).replace(/[^a-zA-Z0-9._-]/g, "_");
3061
- writeFileSync(join(dir, safeName), buffer);
3062
- res.json({
3063
- success: true,
3064
- path: join(dir, safeName),
3065
- filename: safeName,
3066
- size: buffer.length,
3067
- });
3068
- }
3069
- catch (e) {
3070
- res
3071
- .status(500)
3072
- .json({ error: "Failed to save frame", details: String(e) });
3073
- }
3074
- });
3075
3351
  // Open project folder in Finder
3076
3352
  app.post("/api/projects/:id/open-folder", (req, res) => {
3077
3353
  const { type } = req.body || {};