@effectnode/media 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/movie-backend/agent/prompt/ltx.txt +20 -0
- package/dist/backend/movie-backend/agent/prompt/script.md +111 -1
- package/dist/backend/movie-backend/core.js +41 -0
- package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
- package/dist/backend/movie-backend/generation-queue.js +75 -8
- package/dist/backend/movie-backend/render-media.d.ts +79 -2
- package/dist/backend/movie-backend/render-media.js +694 -418
- package/frontend/src/movie-app/components/EditorTabs/AdvancedVoiceCloneTab.tsx +366 -0
- package/frontend/src/movie-app/components/EditorTabs/AudioToVideoTab.tsx +406 -0
- package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +47 -0
- package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +155 -1
- package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +33 -13
- package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +26 -5
- package/frontend/src/movie-app/components/EditorTabs/UpscaleTab.tsx +343 -0
- package/frontend/src/movie-app/components/EditorTabs/VoiceCloneTab.tsx +365 -0
- package/frontend/src/movie-app/components/ProjectEditorPage.tsx +130 -125
- package/frontend/src/movie-app/stores/advancedVoiceCloneStore.ts +205 -0
- package/frontend/src/movie-app/stores/aiModelStore.ts +25 -2
- package/frontend/src/movie-app/stores/audioToVideoStore.ts +274 -0
- package/frontend/src/movie-app/stores/generationStore.ts +156 -255
- package/frontend/src/movie-app/stores/movieStudioStore.ts +10 -3
- package/frontend/src/movie-app/stores/queueStore.ts +47 -1
- package/frontend/src/movie-app/stores/upscaleStore.ts +118 -0
- package/frontend/src/movie-app/stores/voiceCloneStore.ts +227 -0
- package/package.json +1 -1
- package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +0 -913
- package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +0 -305
- package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +0 -249
- package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +0 -267
- package/frontend/src/movie-app/lib/batchVoiceStorage.ts +0 -75
- package/frontend/src/movie-app/stores/batchVoiceStore.ts +0 -990
- package/frontend/src/movie-app/stores/sceneVisualStore.ts +0 -251
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync, readdirSync, statSync, unlinkSync, rmSync, copyFileSync, } from "node:fs";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
|
-
import { join, sep } from "node:path";
|
|
3
|
+
import { dirname, join, sep } from "node:path";
|
|
4
4
|
import { spawn, whichSync, mimeType } from "./process.js";
|
|
5
5
|
// Track the currently active spawn process so it can be cancelled
|
|
6
6
|
let activeProc = null;
|
|
@@ -16,7 +16,6 @@ const APP_DATA_DIR = join(homedir(), "media-studio");
|
|
|
16
16
|
const OUTPUT_DIR = join(APP_DATA_DIR, "output");
|
|
17
17
|
const UPLOAD_DIR = join(APP_DATA_DIR, "upload");
|
|
18
18
|
const AGENT_UPLOAD_DIR = join(APP_DATA_DIR, "agent-upload");
|
|
19
|
-
const EXTRACTED_FRAMES_DIR = join(APP_DATA_DIR, "extracted-frames");
|
|
20
19
|
const AGENTS_DIR = join(APP_DATA_DIR, "agents");
|
|
21
20
|
const JSON_DIR = join(APP_DATA_DIR, "json");
|
|
22
21
|
const PYTHON_DIR = join(APP_DATA_DIR, "python-src");
|
|
@@ -25,7 +24,10 @@ const PROJECTS_FILE = join(JSON_DIR, "projects.json");
|
|
|
25
24
|
const CHARACTERS_FILE = join(JSON_DIR, "characters.json");
|
|
26
25
|
const Z_IMAGE_MODEL = "AbstractFramework/z-image-turbo-8bit";
|
|
27
26
|
const FLUX_KLEIN_MODEL = "AbstractFramework/flux.2-klein-4b-8bit";
|
|
27
|
+
const SEEDVR2_MODEL = "AbstractFramework/seedvr2-7b-8bit";
|
|
28
28
|
const MLX_VLM_MODEL = "mlx-community/gemma-4-e4b-it-8bit";
|
|
29
|
+
const LTX_MODEL_HIGH_QUALITY = "dgrauet/ltx-2.3-mlx";
|
|
30
|
+
const LTX_MODEL_STANDARD = "dgrauet/ltx-2.3-mlx-q8";
|
|
29
31
|
const VIDEO_STAGE_FLAGS = {
|
|
30
32
|
distilled: "--distilled",
|
|
31
33
|
"one-stage": "--one-stage",
|
|
@@ -41,6 +43,12 @@ function stageFlagFor(mode) {
|
|
|
41
43
|
? (VIDEO_STAGE_FLAGS[mode] ?? "--distilled")
|
|
42
44
|
: "--distilled";
|
|
43
45
|
}
|
|
46
|
+
/** Resolve a ltx-2-mlx model id, falling back to the q8 standard model. */
|
|
47
|
+
function resolveLtxModel(model) {
|
|
48
|
+
return String(model) === LTX_MODEL_HIGH_QUALITY
|
|
49
|
+
? LTX_MODEL_HIGH_QUALITY
|
|
50
|
+
: LTX_MODEL_STANDARD;
|
|
51
|
+
}
|
|
44
52
|
// ========== Project Helpers ==========
|
|
45
53
|
function ensureDir(dir) {
|
|
46
54
|
if (!existsSync(dir)) {
|
|
@@ -187,22 +195,6 @@ function resolveSafePath(candidate, projectId) {
|
|
|
187
195
|
}
|
|
188
196
|
return null;
|
|
189
197
|
}
|
|
190
|
-
/** Resolve and validate a user-supplied video filename. Only bare .mp4 names in the output dir. */
|
|
191
|
-
function resolveSafeVideoPath(candidate, projectId) {
|
|
192
|
-
const base = candidate.split(/[/\\]/).pop() || candidate;
|
|
193
|
-
if (base !== candidate || base.includes("..") || base.startsWith(".")) {
|
|
194
|
-
return null;
|
|
195
|
-
}
|
|
196
|
-
if (!base.toLowerCase().endsWith(".mp4"))
|
|
197
|
-
return null;
|
|
198
|
-
const candidatePath = join(OUTPUT_DIR, projectId, base);
|
|
199
|
-
if (!existsSync(candidatePath))
|
|
200
|
-
return null;
|
|
201
|
-
const resolved = realpathSync(candidatePath);
|
|
202
|
-
if (isPathAllowed(resolved))
|
|
203
|
-
return resolved;
|
|
204
|
-
return null;
|
|
205
|
-
}
|
|
206
198
|
/**
|
|
207
199
|
* Resolve the `mlxgen` executable installed via `uv tool install --upgrade mlx-gen`.
|
|
208
200
|
* uv tool installs binaries into `~/.local/bin`; fall back to relying on PATH.
|
|
@@ -219,6 +211,50 @@ async function getMlxgenBin() {
|
|
|
219
211
|
}
|
|
220
212
|
return "mlxgen";
|
|
221
213
|
}
|
|
214
|
+
/** Resolve the `dots-tts` executable installed via `uv tool install dots-tts`. */
|
|
215
|
+
async function getDotsTtsBin() {
|
|
216
|
+
const candidates = [
|
|
217
|
+
join(homedir(), ".local", "bin", "dots-tts"),
|
|
218
|
+
"/opt/homebrew/bin/dots-tts",
|
|
219
|
+
"/usr/local/bin/dots-tts",
|
|
220
|
+
];
|
|
221
|
+
for (const p of candidates) {
|
|
222
|
+
if (existsSync(p))
|
|
223
|
+
return p;
|
|
224
|
+
}
|
|
225
|
+
return "dots-tts";
|
|
226
|
+
}
|
|
227
|
+
/** The cloned dots-tts-mlx project directory. */
|
|
228
|
+
const DOTS_TTS_FOLDER = join(APP_DATA_DIR, "python-src", "dots-tts-mlx");
|
|
229
|
+
/** Base directory where dots-tts MLX weights live (inside the project folder). */
|
|
230
|
+
const DOTS_TTS_WEIGHTS_DIR = join(DOTS_TTS_FOLDER, "dots-tts-mlx-weights");
|
|
231
|
+
/**
|
|
232
|
+
* Resolve a dots-tts `--model` value. Accepts `./dots-tts-mlx-weights/int4`
|
|
233
|
+
* (relative to the dots-tts-mlx folder), a bare variant (`int4`), or an
|
|
234
|
+
* explicit path.
|
|
235
|
+
*/
|
|
236
|
+
function resolveDotsTtsModel(model) {
|
|
237
|
+
if (!model)
|
|
238
|
+
return join(DOTS_TTS_WEIGHTS_DIR, "int4");
|
|
239
|
+
if (model.startsWith("./"))
|
|
240
|
+
return join(DOTS_TTS_FOLDER, model.slice(2));
|
|
241
|
+
if (model.includes("/"))
|
|
242
|
+
return model;
|
|
243
|
+
return join(DOTS_TTS_WEIGHTS_DIR, model);
|
|
244
|
+
}
|
|
245
|
+
/** True when the dots-tts `int4` weights have been downloaded. */
|
|
246
|
+
function isDotsTtsModelDownloaded() {
|
|
247
|
+
return existsSync(join(DOTS_TTS_WEIGHTS_DIR, "int4"));
|
|
248
|
+
}
|
|
249
|
+
/** Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel). */
|
|
250
|
+
async function getFfmpegBin() {
|
|
251
|
+
const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
|
|
252
|
+
for (const p of candidates) {
|
|
253
|
+
if (existsSync(p))
|
|
254
|
+
return p;
|
|
255
|
+
}
|
|
256
|
+
return "ffmpeg";
|
|
257
|
+
}
|
|
222
258
|
/** True when the `mlxgen` executable is installed (known paths or PATH). */
|
|
223
259
|
function isMlxgenInstalled() {
|
|
224
260
|
const candidates = [
|
|
@@ -293,18 +329,6 @@ function isMlxVlmInstalled() {
|
|
|
293
329
|
return false;
|
|
294
330
|
}
|
|
295
331
|
}
|
|
296
|
-
/**
|
|
297
|
-
* Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel).
|
|
298
|
-
* Falls back to relying on PATH when neither known location exists.
|
|
299
|
-
*/
|
|
300
|
-
async function getFfmpegBin() {
|
|
301
|
-
const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
|
|
302
|
-
for (const p of candidates) {
|
|
303
|
-
if (existsSync(p))
|
|
304
|
-
return p;
|
|
305
|
-
}
|
|
306
|
-
return "ffmpeg";
|
|
307
|
-
}
|
|
308
332
|
/** Fixed filename mlx_audio.tts.generate writes its output clip as. */
|
|
309
333
|
const TTS_OUTPUT_FILENAME = "audio_000.mp3";
|
|
310
334
|
/**
|
|
@@ -574,6 +598,76 @@ export async function generateSceneVideo(uvPath, projectId, scene, characters, o
|
|
|
574
598
|
url: `/api/files?path=${encodeURIComponent(videoPath)}`,
|
|
575
599
|
};
|
|
576
600
|
}
|
|
601
|
+
/**
|
|
602
|
+
* Generate a video from a project image via LTX-2.3. `imagePath` must be a bare
|
|
603
|
+
* filename previously uploaded/generated for this project. Used by the queue
|
|
604
|
+
* worker for the "Scene Video Generation" tab.
|
|
605
|
+
*/
|
|
606
|
+
export async function generateImageToVideo(uvPath, projectId, params, onLog) {
|
|
607
|
+
const { prompt, imagePath, mode } = params;
|
|
608
|
+
if (!isValidProjectId(projectId))
|
|
609
|
+
return { error: "Invalid project ID" };
|
|
610
|
+
if (!prompt || !prompt.trim())
|
|
611
|
+
return { error: "Prompt is required" };
|
|
612
|
+
if (!imagePath)
|
|
613
|
+
return { error: "Image path is required" };
|
|
614
|
+
const resolvedImage = resolveSafePath(imagePath, projectId);
|
|
615
|
+
if (!resolvedImage) {
|
|
616
|
+
return {
|
|
617
|
+
error: "Invalid image path. Provide a filename previously uploaded to this project.",
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
|
|
621
|
+
if (!existsSync(ltxFolder)) {
|
|
622
|
+
return { error: "ltx-2-mlx not found. Run setup first." };
|
|
623
|
+
}
|
|
624
|
+
const projectOutputDir = resolveOutputDir(undefined, projectId);
|
|
625
|
+
if (!projectOutputDir)
|
|
626
|
+
return { error: "Invalid output directory." };
|
|
627
|
+
const outputFile = `video-${Date.now()}.mp4`;
|
|
628
|
+
const outputPath = join(projectOutputDir, outputFile);
|
|
629
|
+
const videoWidth = Number(params.width) || 480;
|
|
630
|
+
const videoHeight = Number(params.height) || 480;
|
|
631
|
+
const videoFrames = Number(params.frames) || 121;
|
|
632
|
+
const videoFps = Number(params.frameRate) || 24;
|
|
633
|
+
const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 30);
|
|
634
|
+
const stage2Steps = Math.max(1, Math.round(Number(params.stage2Steps)) || 3);
|
|
635
|
+
const result = await runCommand([
|
|
636
|
+
uvPath,
|
|
637
|
+
"run",
|
|
638
|
+
"ltx-2-mlx",
|
|
639
|
+
"generate",
|
|
640
|
+
"--model",
|
|
641
|
+
resolveLtxModel(params.model),
|
|
642
|
+
"--prompt",
|
|
643
|
+
prompt.trim(),
|
|
644
|
+
stageFlagFor(mode),
|
|
645
|
+
"--stage1-steps",
|
|
646
|
+
String(stage1Steps),
|
|
647
|
+
"--stage2-steps",
|
|
648
|
+
String(stage2Steps),
|
|
649
|
+
"--frames",
|
|
650
|
+
String(videoFrames),
|
|
651
|
+
"--width",
|
|
652
|
+
String(videoWidth),
|
|
653
|
+
"--height",
|
|
654
|
+
String(videoHeight),
|
|
655
|
+
"--frame-rate",
|
|
656
|
+
String(videoFps),
|
|
657
|
+
"--image",
|
|
658
|
+
resolvedImage,
|
|
659
|
+
"--output",
|
|
660
|
+
outputPath,
|
|
661
|
+
], { cwd: ltxFolder, onLog });
|
|
662
|
+
if (!result.success || !existsSync(outputPath)) {
|
|
663
|
+
return { error: result.output || "Video generation failed" };
|
|
664
|
+
}
|
|
665
|
+
backupFile(outputPath, projectId);
|
|
666
|
+
return {
|
|
667
|
+
filename: outputFile,
|
|
668
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
577
671
|
/** Normalize a value into a safe filesystem slug. */
|
|
578
672
|
function slugify(v) {
|
|
579
673
|
return String(v || "")
|
|
@@ -584,10 +678,11 @@ function slugify(v) {
|
|
|
584
678
|
* Generate a single scene image via fast-image-edit (FLUX.2 Klein), using the
|
|
585
679
|
* already-generated character/place images referenced by the scene's slugs.
|
|
586
680
|
*/
|
|
587
|
-
export async function generateSceneImage(projectId, scene, onLog) {
|
|
681
|
+
export async function generateSceneImage(projectId, scene, steps, onLog) {
|
|
588
682
|
const s = slugify(scene?.slug);
|
|
589
683
|
if (!s)
|
|
590
684
|
return { error: "Invalid scene slug" };
|
|
685
|
+
const stepCount = Math.max(1, Number(steps) || 6);
|
|
591
686
|
const outputDir = join(OUTPUT_DIR, projectId);
|
|
592
687
|
const mlxgen = await getMlxgenBin();
|
|
593
688
|
const refImages = [];
|
|
@@ -611,7 +706,7 @@ export async function generateSceneImage(projectId, scene, onLog) {
|
|
|
611
706
|
const fluxArgs = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
|
|
612
707
|
for (const p of refImages)
|
|
613
708
|
fluxArgs.push("--image", p);
|
|
614
|
-
fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps",
|
|
709
|
+
fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
|
|
615
710
|
const result = await runCommand(fluxArgs, { onLog });
|
|
616
711
|
if (!result.success || !existsSync(sceneImagePath)) {
|
|
617
712
|
return { error: result.output || `Failed to generate scene image ${s}` };
|
|
@@ -622,12 +717,353 @@ export async function generateSceneImage(projectId, scene, onLog) {
|
|
|
622
717
|
url: `/api/files?path=${encodeURIComponent(sceneImagePath)}`,
|
|
623
718
|
};
|
|
624
719
|
}
|
|
720
|
+
/**
|
|
721
|
+
* Upscale/refine a project image via mlxgen (SeedVR2). `imagePath` must be a
|
|
722
|
+
* bare filename previously uploaded/generated for this project. `resolution` is
|
|
723
|
+
* either "1x" (refine at native resolution) or "2048" (upscale to 2048px).
|
|
724
|
+
*/
|
|
725
|
+
export async function generateUpscale(projectId, imagePath, resolution, onLog) {
|
|
726
|
+
if (!isValidProjectId(projectId))
|
|
727
|
+
return { error: "Invalid project ID" };
|
|
728
|
+
if (!imagePath)
|
|
729
|
+
return { error: "Image path is required" };
|
|
730
|
+
const resolvedImage = resolveSafePath(imagePath, projectId);
|
|
731
|
+
if (!resolvedImage) {
|
|
732
|
+
return {
|
|
733
|
+
error: "Invalid image path. Provide a filename previously uploaded to this project.",
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
const target = /^(1x|\d+)$/.test(resolution) ? resolution : "1x";
|
|
737
|
+
const mlxgen = await getMlxgenBin();
|
|
738
|
+
const projectOutputDir = join(OUTPUT_DIR, projectId);
|
|
739
|
+
ensureDir(projectOutputDir);
|
|
740
|
+
const outputFile = `upscale-${target}-${Date.now()}.png`;
|
|
741
|
+
const outputPath = join(projectOutputDir, outputFile);
|
|
742
|
+
const result = await runCommand([
|
|
743
|
+
mlxgen,
|
|
744
|
+
"upscale",
|
|
745
|
+
"--model",
|
|
746
|
+
SEEDVR2_MODEL,
|
|
747
|
+
"--image-path",
|
|
748
|
+
resolvedImage,
|
|
749
|
+
"--resolution",
|
|
750
|
+
target,
|
|
751
|
+
"--seed",
|
|
752
|
+
"42",
|
|
753
|
+
"--mlx-cache-limit-gb",
|
|
754
|
+
"100",
|
|
755
|
+
"--output",
|
|
756
|
+
outputPath,
|
|
757
|
+
], { onLog });
|
|
758
|
+
if (!result.success || !existsSync(outputPath)) {
|
|
759
|
+
return { error: result.output || "Upscale failed" };
|
|
760
|
+
}
|
|
761
|
+
backupFile(outputPath, projectId);
|
|
762
|
+
return {
|
|
763
|
+
filename: outputFile,
|
|
764
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* Clone a reference voice and speak `text` via mlx_audio.tts.generate. `refAudioPath`
|
|
769
|
+
* must be a bare filename previously uploaded to this project. `quality` is "low"
|
|
770
|
+
* or "high" (maps to a TTS model). Output is saved under <output>/voices/.
|
|
771
|
+
*/
|
|
772
|
+
export async function generateVoiceClone(uvPath, projectId, text, refAudioPath, quality, onLog) {
|
|
773
|
+
if (!isValidProjectId(projectId))
|
|
774
|
+
return { error: "Invalid project ID" };
|
|
775
|
+
if (!refAudioPath)
|
|
776
|
+
return { error: "Reference audio is required" };
|
|
777
|
+
// Sanitize the transcript: strip control characters (incl. newlines) and
|
|
778
|
+
// collapse whitespace. spawn() runs with shell:false (array args), so shell
|
|
779
|
+
// metacharacters cannot execute, but this keeps `--text` a single well-formed
|
|
780
|
+
// argument and out of the terminal log.
|
|
781
|
+
const cleanText = text
|
|
782
|
+
.replace(/[\u0000-\u001f\u007f]/g, " ")
|
|
783
|
+
.replace(/\s+/g, " ")
|
|
784
|
+
.trim();
|
|
785
|
+
if (!cleanText)
|
|
786
|
+
return { error: "Text is required" };
|
|
787
|
+
const resolvedRef = resolveSafePath(refAudioPath, projectId);
|
|
788
|
+
if (!resolvedRef) {
|
|
789
|
+
return {
|
|
790
|
+
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
const model = quality === "low" ? TTS_MODELS.low : TTS_MODELS.high;
|
|
794
|
+
const projectOutputDir = resolveOutputDir(undefined, projectId);
|
|
795
|
+
if (!projectOutputDir)
|
|
796
|
+
return { error: "Invalid output directory." };
|
|
797
|
+
const voiceId = `voice-${Date.now()}`;
|
|
798
|
+
const voiceDir = join(projectOutputDir, "voices", voiceId);
|
|
799
|
+
ensureDir(voiceDir);
|
|
800
|
+
const result = await runCommand([
|
|
801
|
+
uvPath,
|
|
802
|
+
"run",
|
|
803
|
+
"mlx_audio.tts.generate",
|
|
804
|
+
"--model",
|
|
805
|
+
model,
|
|
806
|
+
"--text",
|
|
807
|
+
cleanText,
|
|
808
|
+
"--ref_audio",
|
|
809
|
+
resolvedRef,
|
|
810
|
+
"--output",
|
|
811
|
+
voiceDir,
|
|
812
|
+
"--audio_format",
|
|
813
|
+
"mp3",
|
|
814
|
+
"--play",
|
|
815
|
+
"--instruct",
|
|
816
|
+
"slow down speech",
|
|
817
|
+
], { cwd: voiceDir, onLog });
|
|
818
|
+
if (!result.success) {
|
|
819
|
+
return { error: result.output || "Voice generation failed" };
|
|
820
|
+
}
|
|
821
|
+
const path = resolveAudioFile(voiceDir);
|
|
822
|
+
if (!path) {
|
|
823
|
+
return { error: "TTS completed but no audio file was produced" };
|
|
824
|
+
}
|
|
825
|
+
const filename = path.split(sep).pop() || TTS_OUTPUT_FILENAME;
|
|
826
|
+
// Persist a per-voice metadata file so the generated-voice list can be rebuilt
|
|
827
|
+
// from the folder structure (each voice in its own folder with meta.json), with
|
|
828
|
+
// no central JSON index.
|
|
829
|
+
const meta = {
|
|
830
|
+
id: voiceId,
|
|
831
|
+
transcript: cleanText,
|
|
832
|
+
quality,
|
|
833
|
+
refAudioFilename: refAudioPath,
|
|
834
|
+
filename,
|
|
835
|
+
createdAt: new Date().toISOString(),
|
|
836
|
+
};
|
|
837
|
+
writeFileSync(join(voiceDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
838
|
+
return {
|
|
839
|
+
filename,
|
|
840
|
+
url: `/api/files?path=${encodeURIComponent(path)}`,
|
|
841
|
+
};
|
|
842
|
+
}
|
|
843
|
+
/** Find the newest video file (mp4/webm/mov) inside a directory. */
|
|
844
|
+
function findNewestVideo(dir) {
|
|
845
|
+
let entries = [];
|
|
846
|
+
try {
|
|
847
|
+
entries = readdirSync(dir);
|
|
848
|
+
}
|
|
849
|
+
catch {
|
|
850
|
+
return null;
|
|
851
|
+
}
|
|
852
|
+
let newest = null;
|
|
853
|
+
for (const name of entries) {
|
|
854
|
+
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
855
|
+
if (ext !== ".mp4" && ext !== ".webm" && ext !== ".mov")
|
|
856
|
+
continue;
|
|
857
|
+
const full = join(dir, name);
|
|
858
|
+
try {
|
|
859
|
+
const st = statSync(full);
|
|
860
|
+
if (!st.isFile())
|
|
861
|
+
continue;
|
|
862
|
+
if (!newest || st.mtimeMs > newest.mtime) {
|
|
863
|
+
newest = { path: full, mtime: st.mtimeMs };
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
catch {
|
|
867
|
+
// skip unreadable entries
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
return newest ? newest.path : null;
|
|
871
|
+
}
|
|
872
|
+
/**
|
|
873
|
+
* Generate a video from an image + audio via ltx-2-mlx `a2v` (audio-to-video).
|
|
874
|
+
* `imagePath` and `audioPath` must be bare filenames previously uploaded/generated
|
|
875
|
+
* for this project. `stage1Steps` maps to `--stage1-steps` (15 default, 30 HD).
|
|
876
|
+
*/
|
|
877
|
+
export async function generateAudioToVideo(uvPath, projectId, params, onLog) {
|
|
878
|
+
if (!isValidProjectId(projectId))
|
|
879
|
+
return { error: "Invalid project ID" };
|
|
880
|
+
if (!params.imagePath)
|
|
881
|
+
return { error: "Image path is required" };
|
|
882
|
+
if (!params.audioPath)
|
|
883
|
+
return { error: "Audio path is required" };
|
|
884
|
+
const resolvedImage = resolveSafePath(params.imagePath, projectId);
|
|
885
|
+
if (!resolvedImage) {
|
|
886
|
+
return {
|
|
887
|
+
error: "Invalid image path. Provide a filename previously uploaded to this project.",
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const resolvedAudio = resolveSafePath(params.audioPath, projectId);
|
|
891
|
+
if (!resolvedAudio) {
|
|
892
|
+
return {
|
|
893
|
+
error: "Invalid audio path. Provide a filename previously uploaded to this project.",
|
|
894
|
+
};
|
|
895
|
+
}
|
|
896
|
+
// Collapse whitespace so the prompt stays a single well-formed CLI argument.
|
|
897
|
+
const cleanPrompt = String(params.prompt || "")
|
|
898
|
+
.replace(/\s+/g, " ")
|
|
899
|
+
.trim() || "scene";
|
|
900
|
+
const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 15);
|
|
901
|
+
// 1 second = 24 frames, plus a terminal frame (24n + 1).
|
|
902
|
+
const frames = Math.max(1, Math.round(Number(params.frames)) || 25);
|
|
903
|
+
const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
|
|
904
|
+
if (!existsSync(ltxFolder)) {
|
|
905
|
+
return { error: "ltx-2-mlx not found. Run setup first." };
|
|
906
|
+
}
|
|
907
|
+
const outputDir = join(OUTPUT_DIR, projectId, `a2v-${Date.now()}`);
|
|
908
|
+
ensureDir(outputDir);
|
|
909
|
+
const result = await runCommand([
|
|
910
|
+
uvPath,
|
|
911
|
+
"run",
|
|
912
|
+
"ltx-2-mlx",
|
|
913
|
+
"a2v",
|
|
914
|
+
"--image",
|
|
915
|
+
resolvedImage,
|
|
916
|
+
"--audio",
|
|
917
|
+
resolvedAudio,
|
|
918
|
+
"--frame-rate",
|
|
919
|
+
"24",
|
|
920
|
+
"--frames",
|
|
921
|
+
String(frames),
|
|
922
|
+
"--output",
|
|
923
|
+
outputDir,
|
|
924
|
+
"--prompt",
|
|
925
|
+
cleanPrompt,
|
|
926
|
+
"--stage1-steps",
|
|
927
|
+
String(stage1Steps),
|
|
928
|
+
"--stage2-steps",
|
|
929
|
+
"3",
|
|
930
|
+
], { cwd: ltxFolder, onLog });
|
|
931
|
+
if (!result.success) {
|
|
932
|
+
return { error: result.output || "Audio-to-video generation failed" };
|
|
933
|
+
}
|
|
934
|
+
const videoPath = findNewestVideo(outputDir);
|
|
935
|
+
if (!videoPath) {
|
|
936
|
+
return { error: "a2v completed but no video file was produced" };
|
|
937
|
+
}
|
|
938
|
+
backupFile(videoPath, projectId);
|
|
939
|
+
return {
|
|
940
|
+
filename: videoPath.split(sep).pop() || "a2v.mp4",
|
|
941
|
+
url: `/api/files?path=${encodeURIComponent(videoPath)}`,
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
/**
|
|
945
|
+
* Clone a reference voice via dots-tts. Output is written to
|
|
946
|
+
* <output>/<projectId>/dots-tts/ as `<prefix>_000.wav`.
|
|
947
|
+
*/
|
|
948
|
+
export async function generateAdvancedVoiceClone(projectId, params, onLog) {
|
|
949
|
+
if (!isValidProjectId(projectId))
|
|
950
|
+
return { error: "Invalid project ID" };
|
|
951
|
+
const cleanText = String(params.text || "")
|
|
952
|
+
.replace(/\s+/g, " ")
|
|
953
|
+
.trim();
|
|
954
|
+
if (!cleanText)
|
|
955
|
+
return { error: "Text is required" };
|
|
956
|
+
if (!params.refAudioPath)
|
|
957
|
+
return { error: "Reference audio is required" };
|
|
958
|
+
const resolvedRef = resolveSafePath(params.refAudioPath, projectId);
|
|
959
|
+
if (!resolvedRef) {
|
|
960
|
+
return {
|
|
961
|
+
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
const language = String(params.language || "YUE").trim() || "YUE";
|
|
965
|
+
const prefix = String(params.outPrefix || "voice")
|
|
966
|
+
.replace(/[^a-zA-Z0-9_-]/g, "_")
|
|
967
|
+
.slice(0, 64) || "voice";
|
|
968
|
+
const model = resolveDotsTtsModel(String(params.model || ""));
|
|
969
|
+
// dots-tts expects a WAV reference — convert mp3/other formats to 16-bit PCM WAV.
|
|
970
|
+
let refAudio = resolvedRef;
|
|
971
|
+
if (!resolvedRef.toLowerCase().endsWith(".wav")) {
|
|
972
|
+
const tempDir = join(TEMP_DIR, String(projectId));
|
|
973
|
+
ensureDir(tempDir);
|
|
974
|
+
const convertedRef = join(tempDir, `avc-ref-${Date.now()}.wav`);
|
|
975
|
+
const ffmpegBin = await getFfmpegBin();
|
|
976
|
+
const conv = await runCommand([
|
|
977
|
+
ffmpegBin,
|
|
978
|
+
"-y",
|
|
979
|
+
"-i",
|
|
980
|
+
resolvedRef,
|
|
981
|
+
"-c:a",
|
|
982
|
+
"pcm_s16le",
|
|
983
|
+
"-ar",
|
|
984
|
+
"44100",
|
|
985
|
+
"-ac",
|
|
986
|
+
"2",
|
|
987
|
+
convertedRef,
|
|
988
|
+
], { onLog });
|
|
989
|
+
if (!conv.success || !existsSync(convertedRef)) {
|
|
990
|
+
return { error: "Failed to convert reference audio to WAV" };
|
|
991
|
+
}
|
|
992
|
+
refAudio = convertedRef;
|
|
993
|
+
}
|
|
994
|
+
const dotsTtsBin = await getDotsTtsBin();
|
|
995
|
+
// Mirror the voice-clone tab's timestamped-folder layout so outputs never
|
|
996
|
+
// overwrite each other.
|
|
997
|
+
const voiceId = `voice-${Date.now()}`;
|
|
998
|
+
const outDir = join(OUTPUT_DIR, projectId, "dots-tts", voiceId);
|
|
999
|
+
ensureDir(outDir);
|
|
1000
|
+
console.log("debug-info:", [
|
|
1001
|
+
dotsTtsBin,
|
|
1002
|
+
"--model",
|
|
1003
|
+
model,
|
|
1004
|
+
"--text",
|
|
1005
|
+
cleanText || "please provide text",
|
|
1006
|
+
"--ref-audio",
|
|
1007
|
+
refAudio,
|
|
1008
|
+
"--language",
|
|
1009
|
+
language,
|
|
1010
|
+
"--out-path",
|
|
1011
|
+
outDir,
|
|
1012
|
+
"--out-prefix",
|
|
1013
|
+
prefix,
|
|
1014
|
+
"--max-generate-length",
|
|
1015
|
+
"3000",
|
|
1016
|
+
]);
|
|
1017
|
+
const result = await runCommand([
|
|
1018
|
+
dotsTtsBin,
|
|
1019
|
+
"--model",
|
|
1020
|
+
model,
|
|
1021
|
+
"--text",
|
|
1022
|
+
cleanText || "please provide text",
|
|
1023
|
+
"--ref-audio",
|
|
1024
|
+
refAudio,
|
|
1025
|
+
"--language",
|
|
1026
|
+
language,
|
|
1027
|
+
"--out-path",
|
|
1028
|
+
outDir,
|
|
1029
|
+
"--out-prefix",
|
|
1030
|
+
prefix,
|
|
1031
|
+
"--max-generate-length",
|
|
1032
|
+
"3000",
|
|
1033
|
+
], { cwd: dirname(dotsTtsBin), onLog });
|
|
1034
|
+
if (!result.success) {
|
|
1035
|
+
return { error: result.output || "Advanced voice clone failed" };
|
|
1036
|
+
}
|
|
1037
|
+
const outputFile = `${prefix}_000.wav`;
|
|
1038
|
+
const outputPath = join(outDir, outputFile);
|
|
1039
|
+
if (!existsSync(outputPath)) {
|
|
1040
|
+
return { error: `Expected output ${outputFile} was not produced` };
|
|
1041
|
+
}
|
|
1042
|
+
// Mirror the voice-clone tab's on-disk conventions: a timestamped backup copy
|
|
1043
|
+
// plus a per-voice meta.json so the output can be listed without a central index.
|
|
1044
|
+
backupFile(outputPath, projectId);
|
|
1045
|
+
const meta = {
|
|
1046
|
+
id: voiceId,
|
|
1047
|
+
transcript: cleanText,
|
|
1048
|
+
language,
|
|
1049
|
+
model,
|
|
1050
|
+
refAudioFilename: params.refAudioPath,
|
|
1051
|
+
outPrefix: prefix,
|
|
1052
|
+
filename: outputFile,
|
|
1053
|
+
createdAt: new Date().toISOString(),
|
|
1054
|
+
};
|
|
1055
|
+
writeFileSync(join(outDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
1056
|
+
return {
|
|
1057
|
+
filename: outputFile,
|
|
1058
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
625
1061
|
/**
|
|
626
1062
|
* Generate a composite image via fast-image-edit (FLUX.2 Klein). `images` are
|
|
627
1063
|
* base64 data URLs that are decoded into temp files and passed to the model as
|
|
628
1064
|
* separate `--image` inputs. Used by the generation queue worker.
|
|
629
1065
|
*/
|
|
630
|
-
export async function generateFastImageEditImage(projectId, prompt, images, onLog) {
|
|
1066
|
+
export async function generateFastImageEditImage(projectId, prompt, images, steps, upscaleResolution, onLog) {
|
|
631
1067
|
if (!isValidProjectId(projectId))
|
|
632
1068
|
return { error: "Invalid project ID" };
|
|
633
1069
|
if (!prompt || !prompt.trim())
|
|
@@ -635,6 +1071,7 @@ export async function generateFastImageEditImage(projectId, prompt, images, onLo
|
|
|
635
1071
|
if (!Array.isArray(images) || images.length === 0) {
|
|
636
1072
|
return { error: "At least one reference image is required" };
|
|
637
1073
|
}
|
|
1074
|
+
const stepCount = Math.max(1, Number(steps) || 4);
|
|
638
1075
|
// Decode each base64 reference image into a temp workspace file so the
|
|
639
1076
|
// FLUX model receives them as separate `--image` inputs.
|
|
640
1077
|
const tempDir = join(TEMP_DIR, String(projectId));
|
|
@@ -661,12 +1098,21 @@ export async function generateFastImageEditImage(projectId, prompt, images, onLo
|
|
|
661
1098
|
const args = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
|
|
662
1099
|
for (const path of tempImagePaths)
|
|
663
1100
|
args.push("--image", path);
|
|
664
|
-
args.push("--prompt", prompt.trim(), "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps",
|
|
1101
|
+
args.push("--prompt", prompt.trim(), "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
|
|
665
1102
|
const result = await runCommand(args, { onLog });
|
|
666
1103
|
if (!result.success || !existsSync(outputPath)) {
|
|
667
1104
|
return { error: result.output || "Fast image edit failed" };
|
|
668
1105
|
}
|
|
669
1106
|
backupFile(outputPath, projectId);
|
|
1107
|
+
// Optionally upscale the generated result (1x / 1500px / 2000px).
|
|
1108
|
+
if (upscaleResolution && upscaleResolution !== "none") {
|
|
1109
|
+
if (onLog)
|
|
1110
|
+
onLog(`Upscaling result (${upscaleResolution})…\n`);
|
|
1111
|
+
const upscaled = await generateUpscale(projectId, outputFile, upscaleResolution, onLog);
|
|
1112
|
+
if ("error" in upscaled)
|
|
1113
|
+
return { error: upscaled.error };
|
|
1114
|
+
return upscaled;
|
|
1115
|
+
}
|
|
670
1116
|
return {
|
|
671
1117
|
filename: outputFile,
|
|
672
1118
|
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
@@ -973,6 +1419,60 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
973
1419
|
raw.sort((a, b) => b.birthtime - a.birthtime);
|
|
974
1420
|
res.json(raw.map(({ filename, url }) => ({ filename, url })));
|
|
975
1421
|
});
|
|
1422
|
+
// List previously generated voice clones by scanning the voices folder. Each
|
|
1423
|
+
// voice lives in its own subfolder with a meta.json (no central JSON index).
|
|
1424
|
+
app.get("/api/projects/:id/voices", (req, res) => {
|
|
1425
|
+
const { id } = req.params;
|
|
1426
|
+
if (!isValidProjectId(id)) {
|
|
1427
|
+
res.status(400).json({ error: "Invalid project ID" });
|
|
1428
|
+
return;
|
|
1429
|
+
}
|
|
1430
|
+
const voicesRoot = join(OUTPUT_DIR, id, "voices");
|
|
1431
|
+
const results = [];
|
|
1432
|
+
if (existsSync(voicesRoot)) {
|
|
1433
|
+
let names = [];
|
|
1434
|
+
try {
|
|
1435
|
+
names = readdirSync(voicesRoot);
|
|
1436
|
+
}
|
|
1437
|
+
catch {
|
|
1438
|
+
names = [];
|
|
1439
|
+
}
|
|
1440
|
+
for (const name of names) {
|
|
1441
|
+
const voiceDir = join(voicesRoot, name);
|
|
1442
|
+
let isDir = false;
|
|
1443
|
+
try {
|
|
1444
|
+
isDir = statSync(voiceDir).isDirectory();
|
|
1445
|
+
}
|
|
1446
|
+
catch {
|
|
1447
|
+
isDir = false;
|
|
1448
|
+
}
|
|
1449
|
+
if (!isDir)
|
|
1450
|
+
continue;
|
|
1451
|
+
let meta = null;
|
|
1452
|
+
try {
|
|
1453
|
+
meta = JSON.parse(readFileSync(join(voiceDir, "meta.json"), "utf-8"));
|
|
1454
|
+
}
|
|
1455
|
+
catch {
|
|
1456
|
+
continue; // folder without a meta.json — skip
|
|
1457
|
+
}
|
|
1458
|
+
const filename = String(meta?.filename || TTS_OUTPUT_FILENAME);
|
|
1459
|
+
const audioPath = join(voiceDir, filename);
|
|
1460
|
+
if (!existsSync(audioPath))
|
|
1461
|
+
continue;
|
|
1462
|
+
results.push({
|
|
1463
|
+
id: name,
|
|
1464
|
+
transcript: String(meta?.transcript ?? ""),
|
|
1465
|
+
quality: String(meta?.quality ?? "high"),
|
|
1466
|
+
refAudioFilename: meta?.refAudioFilename ?? null,
|
|
1467
|
+
filename,
|
|
1468
|
+
createdAt: meta?.createdAt ?? null,
|
|
1469
|
+
url: `/api/files?path=${encodeURIComponent(audioPath)}`,
|
|
1470
|
+
});
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
results.sort((a, b) => String(b.createdAt ?? "").localeCompare(String(a.createdAt ?? "")));
|
|
1474
|
+
res.json(results);
|
|
1475
|
+
});
|
|
976
1476
|
// ========== Render: Text-to-Image ==========
|
|
977
1477
|
app.post("/api/render/text-to-image", async (req, res) => {
|
|
978
1478
|
const { prompt, projectId, width = 512, height = 512, device = "mps", } = req.body || {};
|
|
@@ -1066,7 +1566,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1066
1566
|
});
|
|
1067
1567
|
// ========== Render: Image-to-Video ==========
|
|
1068
1568
|
app.post("/api/render/image-to-video", async (req, res) => {
|
|
1069
|
-
const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", } = req.body || {};
|
|
1569
|
+
const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", stage1Steps, stage2Steps, model, } = req.body || {};
|
|
1070
1570
|
if (!prompt) {
|
|
1071
1571
|
res.status(400).json({ error: "Prompt is required" });
|
|
1072
1572
|
return;
|
|
@@ -1122,6 +1622,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1122
1622
|
const videoHeight = Number(height) || 480;
|
|
1123
1623
|
const videoFrames = Number(frames) || 121;
|
|
1124
1624
|
const videoFps = Number(frameRate) || 24;
|
|
1625
|
+
const stage1 = Math.max(1, Math.round(Number(stage1Steps)) || 30);
|
|
1626
|
+
const stage2 = Math.max(1, Math.round(Number(stage2Steps)) || 3);
|
|
1125
1627
|
send("progress", {
|
|
1126
1628
|
status: "starting",
|
|
1127
1629
|
label: "Generating video...",
|
|
@@ -1139,10 +1641,14 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1139
1641
|
"ltx-2-mlx",
|
|
1140
1642
|
"generate",
|
|
1141
1643
|
"--model",
|
|
1142
|
-
|
|
1644
|
+
resolveLtxModel(model),
|
|
1143
1645
|
"--prompt",
|
|
1144
1646
|
prompt,
|
|
1145
1647
|
stageFlagFor(mode),
|
|
1648
|
+
"--stage1-steps",
|
|
1649
|
+
String(stage1),
|
|
1650
|
+
"--stage2-steps",
|
|
1651
|
+
String(stage2),
|
|
1146
1652
|
// "--low-ram",
|
|
1147
1653
|
"--frames",
|
|
1148
1654
|
String(videoFrames),
|
|
@@ -1738,26 +2244,39 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1738
2244
|
res.end();
|
|
1739
2245
|
}
|
|
1740
2246
|
});
|
|
1741
|
-
// ==========
|
|
1742
|
-
app.
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
2247
|
+
// ========== MLX-Audio: Status ==========
|
|
2248
|
+
app.get("/api/mlxaudio/status", async (_req, res) => {
|
|
2249
|
+
// Never throw on a missing uv — installation state is based on the folder.
|
|
2250
|
+
try {
|
|
2251
|
+
await getUvPath();
|
|
2252
|
+
}
|
|
2253
|
+
catch {
|
|
2254
|
+
// uv not found; installed is still reported from the folder below.
|
|
2255
|
+
}
|
|
2256
|
+
res.json({
|
|
2257
|
+
installed: existsSync(join(PYTHON_DIR, "mlx-audio")),
|
|
2258
|
+
});
|
|
2259
|
+
});
|
|
2260
|
+
// ========== Render: Voice Chat (TTS with reference voice) ==========
|
|
2261
|
+
app.post("/api/render/voice-chat", async (req, res) => {
|
|
2262
|
+
const { text, refAudioPath, projectId } = req.body || {};
|
|
2263
|
+
if (!text || typeof text !== "string" || !text.trim()) {
|
|
2264
|
+
res.status(400).json({ error: "Text is required" });
|
|
1746
2265
|
return;
|
|
1747
2266
|
}
|
|
1748
|
-
if (!
|
|
1749
|
-
res.status(400).json({ error: "
|
|
2267
|
+
if (!refAudioPath) {
|
|
2268
|
+
res.status(400).json({ error: "Reference audio is required" });
|
|
1750
2269
|
return;
|
|
1751
2270
|
}
|
|
1752
|
-
if (!projectId) {
|
|
1753
|
-
res.status(400).json({ error: "
|
|
2271
|
+
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
2272
|
+
res.status(400).json({ error: "Invalid project ID" });
|
|
1754
2273
|
return;
|
|
1755
2274
|
}
|
|
1756
|
-
// Resolve
|
|
1757
|
-
const
|
|
1758
|
-
if (!
|
|
2275
|
+
// Resolve reference audio — only bare filenames in this project's dirs.
|
|
2276
|
+
const resolvedRef = resolveSafePath(refAudioPath, String(projectId));
|
|
2277
|
+
if (!resolvedRef) {
|
|
1759
2278
|
res.status(400).json({
|
|
1760
|
-
error: "Invalid
|
|
2279
|
+
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
1761
2280
|
});
|
|
1762
2281
|
return;
|
|
1763
2282
|
}
|
|
@@ -1772,59 +2291,65 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1772
2291
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
1773
2292
|
};
|
|
1774
2293
|
try {
|
|
1775
|
-
const
|
|
1776
|
-
|
|
1777
|
-
|
|
2294
|
+
const uvPath = await getUvPath();
|
|
2295
|
+
const projectOutputDir = resolveOutputDir(null, String(projectId));
|
|
2296
|
+
if (!projectOutputDir) {
|
|
2297
|
+
send("error", { error: "Invalid output directory." });
|
|
1778
2298
|
res.end();
|
|
1779
2299
|
return;
|
|
1780
2300
|
}
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
const outputPath = join(projectOutputDir, outputFile);
|
|
1786
|
-
const framesToAdd = Number(extendFrames) || 2;
|
|
2301
|
+
// Isolate each utterance so a new response never overwrites a previous
|
|
2302
|
+
// one. mlx_audio writes the clip as <dir>/audio_000.mp3.
|
|
2303
|
+
const voiceDir = join(projectOutputDir, "voice-chat", `chat-${Date.now()}`);
|
|
2304
|
+
ensureDir(voiceDir);
|
|
1787
2305
|
send("progress", {
|
|
1788
2306
|
status: "starting",
|
|
1789
|
-
label: "
|
|
1790
|
-
inputFile: resolvedVideo,
|
|
1791
|
-
outputFile,
|
|
1792
|
-
settings: { extendFrames: framesToAdd },
|
|
2307
|
+
label: "Generating voice...",
|
|
1793
2308
|
});
|
|
1794
2309
|
const proc = spawn([
|
|
1795
2310
|
uvPath,
|
|
1796
2311
|
"run",
|
|
1797
|
-
"
|
|
1798
|
-
"extend",
|
|
2312
|
+
"mlx_audio.tts.generate",
|
|
1799
2313
|
"--model",
|
|
1800
|
-
|
|
1801
|
-
"--
|
|
1802
|
-
|
|
1803
|
-
"--
|
|
1804
|
-
|
|
1805
|
-
"--
|
|
1806
|
-
String(framesToAdd),
|
|
2314
|
+
TTS_MODELS.high,
|
|
2315
|
+
"--text",
|
|
2316
|
+
text.trim(),
|
|
2317
|
+
"--ref_audio",
|
|
2318
|
+
resolvedRef,
|
|
2319
|
+
// "--play",
|
|
1807
2320
|
"--output",
|
|
1808
|
-
|
|
2321
|
+
voiceDir,
|
|
2322
|
+
"--audio_format",
|
|
2323
|
+
"mp3",
|
|
2324
|
+
// "--stream",
|
|
2325
|
+
// "--save",
|
|
2326
|
+
"--instruct",
|
|
2327
|
+
"slow down",
|
|
1809
2328
|
], {
|
|
1810
2329
|
env: process.env,
|
|
1811
|
-
cwd:
|
|
2330
|
+
cwd: voiceDir,
|
|
1812
2331
|
stdout: "pipe",
|
|
1813
2332
|
stderr: "pipe",
|
|
1814
2333
|
});
|
|
1815
2334
|
activeProc = proc;
|
|
1816
|
-
|
|
1817
|
-
const
|
|
1818
|
-
const stderrText = await streamToSSE(proc.stderr, "Extend", send);
|
|
2335
|
+
const stdoutPromise = streamToSSE(proc.stdout, "VoiceChat", send);
|
|
2336
|
+
const stderrText = await streamToSSE(proc.stderr, "VoiceChat", send);
|
|
1819
2337
|
await stdoutPromise;
|
|
1820
2338
|
const exitCode = await proc.exited;
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
2339
|
+
if (exitCode === 0) {
|
|
2340
|
+
const path = resolveAudioFile(voiceDir);
|
|
2341
|
+
if (path) {
|
|
2342
|
+
send("complete", {
|
|
2343
|
+
success: true,
|
|
2344
|
+
path,
|
|
2345
|
+
filename: path.split(sep).pop(),
|
|
2346
|
+
});
|
|
2347
|
+
}
|
|
2348
|
+
else {
|
|
2349
|
+
send("error", {
|
|
2350
|
+
error: "TTS completed but no audio file was produced",
|
|
2351
|
+
});
|
|
2352
|
+
}
|
|
1828
2353
|
}
|
|
1829
2354
|
else {
|
|
1830
2355
|
send("error", {
|
|
@@ -1841,47 +2366,17 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1841
2366
|
res.end();
|
|
1842
2367
|
}
|
|
1843
2368
|
});
|
|
1844
|
-
// ========== MLX-
|
|
1845
|
-
app.get("/api/
|
|
1846
|
-
// Never throw on a missing uv — installation state is based on the folder.
|
|
1847
|
-
try {
|
|
1848
|
-
await getUvPath();
|
|
1849
|
-
}
|
|
1850
|
-
catch {
|
|
1851
|
-
// uv not found; installed is still reported from the folder below.
|
|
1852
|
-
}
|
|
2369
|
+
// ========== MLX-Gen: Status ==========
|
|
2370
|
+
app.get("/api/mlxgen/status", (_req, res) => {
|
|
1853
2371
|
res.json({
|
|
1854
|
-
installed:
|
|
2372
|
+
installed: isMlxgenInstalled(),
|
|
2373
|
+
zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
|
|
2374
|
+
fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
|
|
2375
|
+
seedvr2Downloaded: isModelDownloaded(SEEDVR2_MODEL),
|
|
1855
2376
|
});
|
|
1856
2377
|
});
|
|
1857
|
-
// ==========
|
|
1858
|
-
app.post("/api/
|
|
1859
|
-
const { text, refAudioPath, projectId, outputDir, quality, voiceId } = req.body || {};
|
|
1860
|
-
if (!text || typeof text !== "string" || !text.trim()) {
|
|
1861
|
-
res.status(400).json({ error: "Text is required" });
|
|
1862
|
-
return;
|
|
1863
|
-
}
|
|
1864
|
-
if (!refAudioPath) {
|
|
1865
|
-
res.status(400).json({ error: "Reference audio is required" });
|
|
1866
|
-
return;
|
|
1867
|
-
}
|
|
1868
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
1869
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
1870
|
-
return;
|
|
1871
|
-
}
|
|
1872
|
-
if (quality !== "low" && quality !== "high") {
|
|
1873
|
-
res.status(400).json({ error: "Quality must be 'low' or 'high'" });
|
|
1874
|
-
return;
|
|
1875
|
-
}
|
|
1876
|
-
// Resolve reference audio — only bare filenames in this project's dirs
|
|
1877
|
-
const resolvedRef = resolveSafePath(refAudioPath, projectId);
|
|
1878
|
-
if (!resolvedRef) {
|
|
1879
|
-
res.status(400).json({
|
|
1880
|
-
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
1881
|
-
});
|
|
1882
|
-
return;
|
|
1883
|
-
}
|
|
1884
|
-
// SSE headers
|
|
2378
|
+
// ========== MLX-Gen: Install ==========
|
|
2379
|
+
app.post("/api/mlxgen/install", async (_req, res) => {
|
|
1885
2380
|
res.writeHead(200, {
|
|
1886
2381
|
"Content-Type": "text/event-stream",
|
|
1887
2382
|
"Cache-Control": "no-cache",
|
|
@@ -1893,81 +2388,21 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1893
2388
|
};
|
|
1894
2389
|
try {
|
|
1895
2390
|
const uvPath = await getUvPath();
|
|
1896
|
-
const projectOutputDir = resolveOutputDir(outputDir, projectId);
|
|
1897
|
-
if (!projectOutputDir) {
|
|
1898
|
-
send("error", { error: "Invalid output directory." });
|
|
1899
|
-
res.end();
|
|
1900
|
-
return;
|
|
1901
|
-
}
|
|
1902
|
-
// TTS audio is saved under <projectOutputDir>/voices/<voiceId>/ so each
|
|
1903
|
-
// row's voiceover stays isolated in its own folder. Sanitize voiceId to
|
|
1904
|
-
// alphanumerics/_/- so it can never escape the voices directory.
|
|
1905
|
-
const safeVoiceId = String(voiceId ?? `voice-${Date.now()}`)
|
|
1906
|
-
.replace(/[^a-zA-Z0-9_-]/g, "_")
|
|
1907
|
-
.slice(0, 64);
|
|
1908
|
-
const voiceDir = join(projectOutputDir, "voices", safeVoiceId);
|
|
1909
|
-
ensureDir(voiceDir);
|
|
1910
|
-
// Regenerating a row must overwrite the previous clip: delete any
|
|
1911
|
-
// existing output so the generator writes a fresh audio_000.mp3.
|
|
1912
|
-
const prevAudio = join(voiceDir, TTS_OUTPUT_FILENAME);
|
|
1913
|
-
if (existsSync(prevAudio)) {
|
|
1914
|
-
try {
|
|
1915
|
-
unlinkSync(prevAudio);
|
|
1916
|
-
}
|
|
1917
|
-
catch {
|
|
1918
|
-
// Ignore — the generator will overwrite the file regardless.
|
|
1919
|
-
}
|
|
1920
|
-
}
|
|
1921
2391
|
send("progress", {
|
|
1922
2392
|
status: "starting",
|
|
1923
|
-
label: "
|
|
1924
|
-
model: TTS_MODELS[quality],
|
|
2393
|
+
label: "Installing mlx-gen...",
|
|
1925
2394
|
});
|
|
1926
|
-
|
|
1927
|
-
// lands in <projectOutputDir>/voices/<voiceId>/, which is servable via
|
|
1928
|
-
// /api/files and resolvable via resolveSafePath for the mux step.
|
|
1929
|
-
const proc = spawn([
|
|
1930
|
-
uvPath,
|
|
1931
|
-
"run",
|
|
1932
|
-
"mlx_audio.tts.generate",
|
|
1933
|
-
"--model",
|
|
1934
|
-
TTS_MODELS[quality],
|
|
1935
|
-
"--text",
|
|
1936
|
-
text,
|
|
1937
|
-
"--ref_audio",
|
|
1938
|
-
resolvedRef,
|
|
1939
|
-
"--output",
|
|
1940
|
-
voiceDir,
|
|
1941
|
-
"--audio_format",
|
|
1942
|
-
"mp3",
|
|
1943
|
-
"--play",
|
|
1944
|
-
"--instruct",
|
|
1945
|
-
"slow down speech",
|
|
1946
|
-
], {
|
|
1947
|
-
env: process.env,
|
|
1948
|
-
cwd: voiceDir,
|
|
2395
|
+
const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
|
|
1949
2396
|
stdout: "pipe",
|
|
1950
2397
|
stderr: "pipe",
|
|
1951
2398
|
});
|
|
1952
2399
|
activeProc = proc;
|
|
1953
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
1954
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2400
|
+
const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
|
|
2401
|
+
const stderrText = await streamToSSE(proc.stderr, "Install", send);
|
|
1955
2402
|
await stdoutPromise;
|
|
1956
2403
|
const exitCode = await proc.exited;
|
|
1957
2404
|
if (exitCode === 0) {
|
|
1958
|
-
|
|
1959
|
-
if (path) {
|
|
1960
|
-
send("complete", {
|
|
1961
|
-
success: true,
|
|
1962
|
-
path,
|
|
1963
|
-
filename: path.split(sep).pop(),
|
|
1964
|
-
});
|
|
1965
|
-
}
|
|
1966
|
-
else {
|
|
1967
|
-
send("error", {
|
|
1968
|
-
error: "TTS completed but no audio file was produced",
|
|
1969
|
-
});
|
|
1970
|
-
}
|
|
2405
|
+
send("complete", { success: true });
|
|
1971
2406
|
}
|
|
1972
2407
|
else {
|
|
1973
2408
|
send("error", {
|
|
@@ -1984,30 +2419,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1984
2419
|
res.end();
|
|
1985
2420
|
}
|
|
1986
2421
|
});
|
|
1987
|
-
// ==========
|
|
1988
|
-
app.post("/api/
|
|
1989
|
-
const { text, refAudioPath, projectId } = req.body || {};
|
|
1990
|
-
if (!text || typeof text !== "string" || !text.trim()) {
|
|
1991
|
-
res.status(400).json({ error: "Text is required" });
|
|
1992
|
-
return;
|
|
1993
|
-
}
|
|
1994
|
-
if (!refAudioPath) {
|
|
1995
|
-
res.status(400).json({ error: "Reference audio is required" });
|
|
1996
|
-
return;
|
|
1997
|
-
}
|
|
1998
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
1999
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
2000
|
-
return;
|
|
2001
|
-
}
|
|
2002
|
-
// Resolve reference audio — only bare filenames in this project's dirs.
|
|
2003
|
-
const resolvedRef = resolveSafePath(refAudioPath, String(projectId));
|
|
2004
|
-
if (!resolvedRef) {
|
|
2005
|
-
res.status(400).json({
|
|
2006
|
-
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
2007
|
-
});
|
|
2008
|
-
return;
|
|
2009
|
-
}
|
|
2010
|
-
// SSE headers
|
|
2422
|
+
// ========== MLX-Gen: Download Z-Image Model ==========
|
|
2423
|
+
app.post("/api/mlxgen/download-z-model", async (_req, res) => {
|
|
2011
2424
|
res.writeHead(200, {
|
|
2012
2425
|
"Content-Type": "text/event-stream",
|
|
2013
2426
|
"Cache-Control": "no-cache",
|
|
@@ -2017,66 +2430,24 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2017
2430
|
const send = (event, data) => {
|
|
2018
2431
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2019
2432
|
};
|
|
2433
|
+
const model = Z_IMAGE_MODEL;
|
|
2020
2434
|
try {
|
|
2021
|
-
const
|
|
2022
|
-
const projectOutputDir = resolveOutputDir(null, String(projectId));
|
|
2023
|
-
if (!projectOutputDir) {
|
|
2024
|
-
send("error", { error: "Invalid output directory." });
|
|
2025
|
-
res.end();
|
|
2026
|
-
return;
|
|
2027
|
-
}
|
|
2028
|
-
// Isolate each utterance so a new response never overwrites a previous
|
|
2029
|
-
// one. mlx_audio writes the clip as <dir>/audio_000.mp3.
|
|
2030
|
-
const voiceDir = join(projectOutputDir, "voice-chat", `chat-${Date.now()}`);
|
|
2031
|
-
ensureDir(voiceDir);
|
|
2435
|
+
const mlxgen = await getMlxgenBin();
|
|
2032
2436
|
send("progress", {
|
|
2033
2437
|
status: "starting",
|
|
2034
|
-
label:
|
|
2438
|
+
label: `Downloading model ${model}...`,
|
|
2035
2439
|
});
|
|
2036
|
-
const proc = spawn([
|
|
2037
|
-
uvPath,
|
|
2038
|
-
"run",
|
|
2039
|
-
"mlx_audio.tts.generate",
|
|
2040
|
-
"--model",
|
|
2041
|
-
TTS_MODELS.high,
|
|
2042
|
-
"--text",
|
|
2043
|
-
text.trim(),
|
|
2044
|
-
"--ref_audio",
|
|
2045
|
-
resolvedRef,
|
|
2046
|
-
// "--play",
|
|
2047
|
-
"--output",
|
|
2048
|
-
voiceDir,
|
|
2049
|
-
"--audio_format",
|
|
2050
|
-
"mp3",
|
|
2051
|
-
// "--stream",
|
|
2052
|
-
// "--save",
|
|
2053
|
-
"--instruct",
|
|
2054
|
-
"slow down",
|
|
2055
|
-
], {
|
|
2056
|
-
env: process.env,
|
|
2057
|
-
cwd: voiceDir,
|
|
2440
|
+
const proc = spawn([mlxgen, "download", "--model", model], {
|
|
2058
2441
|
stdout: "pipe",
|
|
2059
2442
|
stderr: "pipe",
|
|
2060
2443
|
});
|
|
2061
2444
|
activeProc = proc;
|
|
2062
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2063
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2445
|
+
const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
|
|
2446
|
+
const stderrText = await streamToSSE(proc.stderr, "Download", send);
|
|
2064
2447
|
await stdoutPromise;
|
|
2065
2448
|
const exitCode = await proc.exited;
|
|
2066
2449
|
if (exitCode === 0) {
|
|
2067
|
-
|
|
2068
|
-
if (path) {
|
|
2069
|
-
send("complete", {
|
|
2070
|
-
success: true,
|
|
2071
|
-
path,
|
|
2072
|
-
filename: path.split(sep).pop(),
|
|
2073
|
-
});
|
|
2074
|
-
}
|
|
2075
|
-
else {
|
|
2076
|
-
send("error", {
|
|
2077
|
-
error: "TTS completed but no audio file was produced",
|
|
2078
|
-
});
|
|
2079
|
-
}
|
|
2450
|
+
send("complete", { success: true });
|
|
2080
2451
|
}
|
|
2081
2452
|
else {
|
|
2082
2453
|
send("error", {
|
|
@@ -2093,38 +2464,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2093
2464
|
res.end();
|
|
2094
2465
|
}
|
|
2095
2466
|
});
|
|
2096
|
-
// ==========
|
|
2097
|
-
app.post("/api/
|
|
2098
|
-
const { videoPath, audioPath, projectId, outputDir } = req.body || {};
|
|
2099
|
-
if (!videoPath) {
|
|
2100
|
-
res.status(400).json({ error: "Video path is required" });
|
|
2101
|
-
return;
|
|
2102
|
-
}
|
|
2103
|
-
if (!audioPath) {
|
|
2104
|
-
res.status(400).json({ error: "Audio path is required" });
|
|
2105
|
-
return;
|
|
2106
|
-
}
|
|
2107
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
2108
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
2109
|
-
return;
|
|
2110
|
-
}
|
|
2111
|
-
// Resolve video path — only bare .mp4 filenames in this project's output dir
|
|
2112
|
-
const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
|
|
2113
|
-
if (!resolvedVideo) {
|
|
2114
|
-
res.status(400).json({
|
|
2115
|
-
error: "Invalid video path. Provide a filename previously generated in this project.",
|
|
2116
|
-
});
|
|
2117
|
-
return;
|
|
2118
|
-
}
|
|
2119
|
-
// Resolve audio path — only bare filenames in this project's dirs
|
|
2120
|
-
const resolvedAudio = resolveSafePath(audioPath, projectId);
|
|
2121
|
-
if (!resolvedAudio) {
|
|
2122
|
-
res.status(400).json({
|
|
2123
|
-
error: "Invalid audio path. Provide a filename previously generated or uploaded to this project.",
|
|
2124
|
-
});
|
|
2125
|
-
return;
|
|
2126
|
-
}
|
|
2127
|
-
// SSE headers
|
|
2467
|
+
// ========== MLX-Gen: Download FLUX.2 Klein Model ==========
|
|
2468
|
+
app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
|
|
2128
2469
|
res.writeHead(200, {
|
|
2129
2470
|
"Content-Type": "text/event-stream",
|
|
2130
2471
|
"Cache-Control": "no-cache",
|
|
@@ -2135,54 +2476,22 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2135
2476
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2136
2477
|
};
|
|
2137
2478
|
try {
|
|
2138
|
-
const
|
|
2139
|
-
const projectOutputDir = resolveOutputDir(outputDir, projectId);
|
|
2140
|
-
if (!projectOutputDir) {
|
|
2141
|
-
send("error", { error: "Invalid output directory." });
|
|
2142
|
-
res.end();
|
|
2143
|
-
return;
|
|
2144
|
-
}
|
|
2145
|
-
const finalFile = `voice-${Date.now()}.mp4`;
|
|
2146
|
-
const finalPath = join(projectOutputDir, finalFile);
|
|
2479
|
+
const mlxgen = await getMlxgenBin();
|
|
2147
2480
|
send("progress", {
|
|
2148
2481
|
status: "starting",
|
|
2149
|
-
label:
|
|
2150
|
-
outputFile: finalFile,
|
|
2482
|
+
label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
|
|
2151
2483
|
});
|
|
2152
|
-
const proc = spawn([
|
|
2153
|
-
ffmpegBin,
|
|
2154
|
-
"-y",
|
|
2155
|
-
"-i",
|
|
2156
|
-
resolvedVideo,
|
|
2157
|
-
"-i",
|
|
2158
|
-
resolvedAudio,
|
|
2159
|
-
"-map",
|
|
2160
|
-
"0:v",
|
|
2161
|
-
"-map",
|
|
2162
|
-
"1:a",
|
|
2163
|
-
"-c:v",
|
|
2164
|
-
"copy",
|
|
2165
|
-
"-c:a",
|
|
2166
|
-
"aac",
|
|
2167
|
-
"-b:a",
|
|
2168
|
-
"192k",
|
|
2169
|
-
"-shortest",
|
|
2170
|
-
finalPath,
|
|
2171
|
-
], {
|
|
2484
|
+
const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
|
|
2172
2485
|
stdout: "pipe",
|
|
2173
2486
|
stderr: "pipe",
|
|
2174
2487
|
});
|
|
2175
2488
|
activeProc = proc;
|
|
2176
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2177
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2489
|
+
const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
|
|
2490
|
+
const stderrText = await streamToSSE(proc.stderr, "Download", send);
|
|
2178
2491
|
await stdoutPromise;
|
|
2179
2492
|
const exitCode = await proc.exited;
|
|
2180
|
-
if (exitCode === 0
|
|
2181
|
-
send("complete", {
|
|
2182
|
-
success: true,
|
|
2183
|
-
path: finalPath,
|
|
2184
|
-
filename: finalFile,
|
|
2185
|
-
});
|
|
2493
|
+
if (exitCode === 0) {
|
|
2494
|
+
send("complete", { success: true });
|
|
2186
2495
|
}
|
|
2187
2496
|
else {
|
|
2188
2497
|
send("error", {
|
|
@@ -2199,16 +2508,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2199
2508
|
res.end();
|
|
2200
2509
|
}
|
|
2201
2510
|
});
|
|
2202
|
-
// ========== MLX-Gen:
|
|
2203
|
-
app.
|
|
2204
|
-
res.json({
|
|
2205
|
-
installed: isMlxgenInstalled(),
|
|
2206
|
-
zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
|
|
2207
|
-
fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
|
|
2208
|
-
});
|
|
2209
|
-
});
|
|
2210
|
-
// ========== MLX-Gen: Install ==========
|
|
2211
|
-
app.post("/api/mlxgen/install", async (_req, res) => {
|
|
2511
|
+
// ========== MLX-Gen: Download SeedVR2 Model ==========
|
|
2512
|
+
app.post("/api/mlxgen/download-seedvr2-model", async (_req, res) => {
|
|
2212
2513
|
res.writeHead(200, {
|
|
2213
2514
|
"Content-Type": "text/event-stream",
|
|
2214
2515
|
"Cache-Control": "no-cache",
|
|
@@ -2219,18 +2520,18 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2219
2520
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2220
2521
|
};
|
|
2221
2522
|
try {
|
|
2222
|
-
const
|
|
2523
|
+
const mlxgen = await getMlxgenBin();
|
|
2223
2524
|
send("progress", {
|
|
2224
2525
|
status: "starting",
|
|
2225
|
-
label:
|
|
2526
|
+
label: `Downloading model ${SEEDVR2_MODEL}...`,
|
|
2226
2527
|
});
|
|
2227
|
-
const proc = spawn([
|
|
2528
|
+
const proc = spawn([mlxgen, "download", "--model", SEEDVR2_MODEL], {
|
|
2228
2529
|
stdout: "pipe",
|
|
2229
2530
|
stderr: "pipe",
|
|
2230
2531
|
});
|
|
2231
2532
|
activeProc = proc;
|
|
2232
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2233
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2533
|
+
const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
|
|
2534
|
+
const stderrText = await streamToSSE(proc.stderr, "Download", send);
|
|
2234
2535
|
await stdoutPromise;
|
|
2235
2536
|
const exitCode = await proc.exited;
|
|
2236
2537
|
if (exitCode === 0) {
|
|
@@ -2251,8 +2552,25 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2251
2552
|
res.end();
|
|
2252
2553
|
}
|
|
2253
2554
|
});
|
|
2254
|
-
// ==========
|
|
2255
|
-
app.
|
|
2555
|
+
// ========== H3: Download Model ==========
|
|
2556
|
+
app.get("/api/h3/status", (_req, res) => {
|
|
2557
|
+
res.json({ downloaded: isH3ModelDownloaded() });
|
|
2558
|
+
});
|
|
2559
|
+
// ========== Hugging Face CLI + LTX Video Model ==========
|
|
2560
|
+
app.get("/api/hf/status", (_req, res) => {
|
|
2561
|
+
res.json({
|
|
2562
|
+
installed: whichSync("hf") !== null,
|
|
2563
|
+
ltxDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx-q8"),
|
|
2564
|
+
ltxBaseDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx"),
|
|
2565
|
+
ttsDownloaded: isModelDownloaded("Qwen/Qwen3-TTS-12Hz-1.7B-Base"),
|
|
2566
|
+
mlxVlmDownloaded: isModelDownloaded(MLX_VLM_MODEL),
|
|
2567
|
+
});
|
|
2568
|
+
});
|
|
2569
|
+
// ========== Dots-TTS: Status + Model Download ==========
|
|
2570
|
+
app.get("/api/dots-tts/status", (_req, res) => {
|
|
2571
|
+
res.json({ downloaded: isDotsTtsModelDownloaded() });
|
|
2572
|
+
});
|
|
2573
|
+
app.post("/api/dots-tts/download-model", async (_req, res) => {
|
|
2256
2574
|
res.writeHead(200, {
|
|
2257
2575
|
"Content-Type": "text/event-stream",
|
|
2258
2576
|
"Cache-Control": "no-cache",
|
|
@@ -2262,20 +2580,24 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2262
2580
|
const send = (event, data) => {
|
|
2263
2581
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2264
2582
|
};
|
|
2265
|
-
const model = Z_IMAGE_MODEL;
|
|
2266
2583
|
try {
|
|
2267
|
-
|
|
2584
|
+
ensureDir(DOTS_TTS_FOLDER);
|
|
2268
2585
|
send("progress", {
|
|
2269
2586
|
status: "starting",
|
|
2270
|
-
label:
|
|
2271
|
-
});
|
|
2272
|
-
const proc = spawn([mlxgen, "download", "--model", model], {
|
|
2273
|
-
stdout: "pipe",
|
|
2274
|
-
stderr: "pipe",
|
|
2587
|
+
label: "Downloading dots-tts model (int4)...",
|
|
2275
2588
|
});
|
|
2589
|
+
const proc = spawn([
|
|
2590
|
+
"hf",
|
|
2591
|
+
"download",
|
|
2592
|
+
"shraey/dots-tts-mlx",
|
|
2593
|
+
"--include",
|
|
2594
|
+
"int4/*",
|
|
2595
|
+
"--local-dir",
|
|
2596
|
+
"./dots-tts-mlx-weights",
|
|
2597
|
+
], { cwd: DOTS_TTS_FOLDER, stdout: "pipe", stderr: "pipe" });
|
|
2276
2598
|
activeProc = proc;
|
|
2277
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2278
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2599
|
+
const stdoutPromise = streamToSSE(proc.stdout, "dots-tts", send);
|
|
2600
|
+
const stderrText = await streamToSSE(proc.stderr, "dots-tts", send);
|
|
2279
2601
|
await stdoutPromise;
|
|
2280
2602
|
const exitCode = await proc.exited;
|
|
2281
2603
|
if (exitCode === 0) {
|
|
@@ -2296,8 +2618,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2296
2618
|
res.end();
|
|
2297
2619
|
}
|
|
2298
2620
|
});
|
|
2299
|
-
|
|
2300
|
-
app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
|
|
2621
|
+
app.post("/api/hf/install", async (_req, res) => {
|
|
2301
2622
|
res.writeHead(200, {
|
|
2302
2623
|
"Content-Type": "text/event-stream",
|
|
2303
2624
|
"Cache-Control": "no-cache",
|
|
@@ -2308,18 +2629,14 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2308
2629
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2309
2630
|
};
|
|
2310
2631
|
try {
|
|
2311
|
-
const mlxgen = await getMlxgenBin();
|
|
2312
2632
|
send("progress", {
|
|
2313
2633
|
status: "starting",
|
|
2314
|
-
label:
|
|
2315
|
-
});
|
|
2316
|
-
const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
|
|
2317
|
-
stdout: "pipe",
|
|
2318
|
-
stderr: "pipe",
|
|
2634
|
+
label: "Installing huggingface-cli...",
|
|
2319
2635
|
});
|
|
2636
|
+
const proc = spawn(["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash"], { stdout: "pipe", stderr: "pipe" });
|
|
2320
2637
|
activeProc = proc;
|
|
2321
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2322
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2638
|
+
const stdoutPromise = streamToSSE(proc.stdout, "HF Install", send);
|
|
2639
|
+
const stderrText = await streamToSSE(proc.stderr, "HF Install", send);
|
|
2323
2640
|
await stdoutPromise;
|
|
2324
2641
|
const exitCode = await proc.exited;
|
|
2325
2642
|
if (exitCode === 0) {
|
|
@@ -2340,20 +2657,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2340
2657
|
res.end();
|
|
2341
2658
|
}
|
|
2342
2659
|
});
|
|
2343
|
-
|
|
2344
|
-
app.get("/api/h3/status", (_req, res) => {
|
|
2345
|
-
res.json({ downloaded: isH3ModelDownloaded() });
|
|
2346
|
-
});
|
|
2347
|
-
// ========== Hugging Face CLI + LTX Video Model ==========
|
|
2348
|
-
app.get("/api/hf/status", (_req, res) => {
|
|
2349
|
-
res.json({
|
|
2350
|
-
installed: whichSync("hf") !== null,
|
|
2351
|
-
ltxDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx-q8"),
|
|
2352
|
-
ttsDownloaded: isModelDownloaded("Qwen/Qwen3-TTS-12Hz-1.7B-Base"),
|
|
2353
|
-
mlxVlmDownloaded: isModelDownloaded(MLX_VLM_MODEL),
|
|
2354
|
-
});
|
|
2355
|
-
});
|
|
2356
|
-
app.post("/api/hf/install", async (_req, res) => {
|
|
2660
|
+
app.post("/api/hf/download-ltx", async (_req, res) => {
|
|
2357
2661
|
res.writeHead(200, {
|
|
2358
2662
|
"Content-Type": "text/event-stream",
|
|
2359
2663
|
"Cache-Control": "no-cache",
|
|
@@ -2366,12 +2670,15 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2366
2670
|
try {
|
|
2367
2671
|
send("progress", {
|
|
2368
2672
|
status: "starting",
|
|
2369
|
-
label: "
|
|
2673
|
+
label: "Downloading dgrauet/ltx-2.3-mlx-q8...",
|
|
2674
|
+
});
|
|
2675
|
+
const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx-q8"], {
|
|
2676
|
+
stdout: "pipe",
|
|
2677
|
+
stderr: "pipe",
|
|
2370
2678
|
});
|
|
2371
|
-
const proc = spawn(["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash"], { stdout: "pipe", stderr: "pipe" });
|
|
2372
2679
|
activeProc = proc;
|
|
2373
|
-
const stdoutPromise = streamToSSE(proc.stdout, "HF
|
|
2374
|
-
const stderrText = await streamToSSE(proc.stderr, "HF
|
|
2680
|
+
const stdoutPromise = streamToSSE(proc.stdout, "HF Download", send);
|
|
2681
|
+
const stderrText = await streamToSSE(proc.stderr, "HF Download", send);
|
|
2375
2682
|
await stdoutPromise;
|
|
2376
2683
|
const exitCode = await proc.exited;
|
|
2377
2684
|
if (exitCode === 0) {
|
|
@@ -2392,7 +2699,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2392
2699
|
res.end();
|
|
2393
2700
|
}
|
|
2394
2701
|
});
|
|
2395
|
-
app.post("/api/hf/download-ltx", async (_req, res) => {
|
|
2702
|
+
app.post("/api/hf/download-ltx-base", async (_req, res) => {
|
|
2396
2703
|
res.writeHead(200, {
|
|
2397
2704
|
"Content-Type": "text/event-stream",
|
|
2398
2705
|
"Cache-Control": "no-cache",
|
|
@@ -2405,9 +2712,9 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2405
2712
|
try {
|
|
2406
2713
|
send("progress", {
|
|
2407
2714
|
status: "starting",
|
|
2408
|
-
label: "Downloading dgrauet/ltx-2.3-mlx
|
|
2715
|
+
label: "Downloading dgrauet/ltx-2.3-mlx...",
|
|
2409
2716
|
});
|
|
2410
|
-
const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx
|
|
2717
|
+
const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx"], {
|
|
2411
2718
|
stdout: "pipe",
|
|
2412
2719
|
stderr: "pipe",
|
|
2413
2720
|
});
|
|
@@ -3041,37 +3348,6 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
3041
3348
|
writeCharacters(characters);
|
|
3042
3349
|
res.json(removed);
|
|
3043
3350
|
});
|
|
3044
|
-
// Save an extracted video frame into the project's extracted-frames folder.
|
|
3045
|
-
app.post("/api/extracted-frames", (req, res) => {
|
|
3046
|
-
const { image, filename, projectId } = req.body || {};
|
|
3047
|
-
if (!image) {
|
|
3048
|
-
res.status(400).json({ error: "Image data is required (base64)" });
|
|
3049
|
-
return;
|
|
3050
|
-
}
|
|
3051
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
3052
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
3053
|
-
return;
|
|
3054
|
-
}
|
|
3055
|
-
try {
|
|
3056
|
-
const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
|
|
3057
|
-
const buffer = Buffer.from(base64, "base64");
|
|
3058
|
-
const dir = join(EXTRACTED_FRAMES_DIR, String(projectId));
|
|
3059
|
-
ensureDir(dir);
|
|
3060
|
-
const safeName = (filename || `frame-${Date.now()}.png`).replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
3061
|
-
writeFileSync(join(dir, safeName), buffer);
|
|
3062
|
-
res.json({
|
|
3063
|
-
success: true,
|
|
3064
|
-
path: join(dir, safeName),
|
|
3065
|
-
filename: safeName,
|
|
3066
|
-
size: buffer.length,
|
|
3067
|
-
});
|
|
3068
|
-
}
|
|
3069
|
-
catch (e) {
|
|
3070
|
-
res
|
|
3071
|
-
.status(500)
|
|
3072
|
-
.json({ error: "Failed to save frame", details: String(e) });
|
|
3073
|
-
}
|
|
3074
|
-
});
|
|
3075
3351
|
// Open project folder in Finder
|
|
3076
3352
|
app.post("/api/projects/:id/open-folder", (req, res) => {
|
|
3077
3353
|
const { type } = req.body || {};
|