@effectnode/media 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/movie-backend/agent/prompt/script.md +213 -0
- package/dist/backend/movie-backend/core.js +33 -0
- package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
- package/dist/backend/movie-backend/generation-queue.js +82 -6
- package/dist/backend/movie-backend/render-media.d.ts +89 -1
- package/dist/backend/movie-backend/render-media.js +772 -476
- package/frontend/src/movie-app/components/EditorTabs/AdvancedVoiceCloneTab.tsx +366 -0
- package/frontend/src/movie-app/components/EditorTabs/AudioToVideoTab.tsx +406 -0
- package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +67 -51
- package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +155 -1
- package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +33 -13
- package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +26 -5
- package/frontend/src/movie-app/components/EditorTabs/UpscaleTab.tsx +342 -0
- package/frontend/src/movie-app/components/EditorTabs/VoiceCloneTab.tsx +365 -0
- package/frontend/src/movie-app/components/ProjectEditorPage.tsx +131 -126
- package/frontend/src/movie-app/stores/advancedVoiceCloneStore.ts +207 -0
- package/frontend/src/movie-app/stores/aiModelStore.ts +25 -2
- package/frontend/src/movie-app/stores/audioToVideoStore.ts +274 -0
- package/frontend/src/movie-app/stores/generationStore.ts +250 -345
- package/frontend/src/movie-app/stores/movieStudioStore.ts +10 -3
- package/frontend/src/movie-app/stores/projectStore.ts +7 -3
- package/frontend/src/movie-app/stores/queueStore.ts +48 -1
- package/frontend/src/movie-app/stores/upscaleStore.ts +118 -0
- package/frontend/src/movie-app/stores/voiceCloneStore.ts +227 -0
- package/package.json +1 -1
- package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +0 -913
- package/frontend/src/movie-app/components/EditorTabs/CharacterSheet.tsx +0 -234
- package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +0 -305
- package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +0 -249
- package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +0 -267
- package/frontend/src/movie-app/lib/batchVoiceStorage.ts +0 -75
- package/frontend/src/movie-app/stores/batchVoiceStore.ts +0 -990
- package/frontend/src/movie-app/stores/sceneVisualStore.ts +0 -251
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync, readdirSync, statSync, unlinkSync, rmSync, copyFileSync, } from "node:fs";
|
|
2
|
-
import { randomUUID } from "node:crypto";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
|
-
import { join, sep } from "node:path";
|
|
3
|
+
import { dirname, join, sep } from "node:path";
|
|
5
4
|
import { spawn, whichSync, mimeType } from "./process.js";
|
|
6
5
|
// Track the currently active spawn process so it can be cancelled
|
|
7
6
|
let activeProc = null;
|
|
@@ -17,8 +16,6 @@ const APP_DATA_DIR = join(homedir(), "media-studio");
|
|
|
17
16
|
const OUTPUT_DIR = join(APP_DATA_DIR, "output");
|
|
18
17
|
const UPLOAD_DIR = join(APP_DATA_DIR, "upload");
|
|
19
18
|
const AGENT_UPLOAD_DIR = join(APP_DATA_DIR, "agent-upload");
|
|
20
|
-
const EXTRACTED_FRAMES_DIR = join(APP_DATA_DIR, "extracted-frames");
|
|
21
|
-
const CHARACTER_SHEET_DIR = join(APP_DATA_DIR, "character-sheet");
|
|
22
19
|
const AGENTS_DIR = join(APP_DATA_DIR, "agents");
|
|
23
20
|
const JSON_DIR = join(APP_DATA_DIR, "json");
|
|
24
21
|
const PYTHON_DIR = join(APP_DATA_DIR, "python-src");
|
|
@@ -27,7 +24,10 @@ const PROJECTS_FILE = join(JSON_DIR, "projects.json");
|
|
|
27
24
|
const CHARACTERS_FILE = join(JSON_DIR, "characters.json");
|
|
28
25
|
const Z_IMAGE_MODEL = "AbstractFramework/z-image-turbo-8bit";
|
|
29
26
|
const FLUX_KLEIN_MODEL = "AbstractFramework/flux.2-klein-4b-8bit";
|
|
27
|
+
const SEEDVR2_MODEL = "AbstractFramework/seedvr2-7b-8bit";
|
|
30
28
|
const MLX_VLM_MODEL = "mlx-community/gemma-4-e4b-it-8bit";
|
|
29
|
+
const LTX_MODEL_HIGH_QUALITY = "dgrauet/ltx-2.3-mlx";
|
|
30
|
+
const LTX_MODEL_STANDARD = "dgrauet/ltx-2.3-mlx-q8";
|
|
31
31
|
const VIDEO_STAGE_FLAGS = {
|
|
32
32
|
distilled: "--distilled",
|
|
33
33
|
"one-stage": "--one-stage",
|
|
@@ -43,6 +43,12 @@ function stageFlagFor(mode) {
|
|
|
43
43
|
? (VIDEO_STAGE_FLAGS[mode] ?? "--distilled")
|
|
44
44
|
: "--distilled";
|
|
45
45
|
}
|
|
46
|
+
/** Resolve a ltx-2-mlx model id, falling back to the q8 standard model. */
|
|
47
|
+
function resolveLtxModel(model) {
|
|
48
|
+
return String(model) === LTX_MODEL_HIGH_QUALITY
|
|
49
|
+
? LTX_MODEL_HIGH_QUALITY
|
|
50
|
+
: LTX_MODEL_STANDARD;
|
|
51
|
+
}
|
|
46
52
|
// ========== Project Helpers ==========
|
|
47
53
|
function ensureDir(dir) {
|
|
48
54
|
if (!existsSync(dir)) {
|
|
@@ -141,12 +147,11 @@ let _allowedRealDirs = null;
|
|
|
141
147
|
function getAllowedRealDirs() {
|
|
142
148
|
if (_allowedRealDirs)
|
|
143
149
|
return _allowedRealDirs;
|
|
144
|
-
[OUTPUT_DIR, UPLOAD_DIR, AGENT_UPLOAD_DIR
|
|
150
|
+
[OUTPUT_DIR, UPLOAD_DIR, AGENT_UPLOAD_DIR].forEach((d) => ensureDir(d));
|
|
145
151
|
_allowedRealDirs = [
|
|
146
152
|
realpathSync(OUTPUT_DIR) + sep,
|
|
147
153
|
realpathSync(UPLOAD_DIR) + sep,
|
|
148
154
|
realpathSync(AGENT_UPLOAD_DIR) + sep,
|
|
149
|
-
realpathSync(CHARACTER_SHEET_DIR) + sep,
|
|
150
155
|
];
|
|
151
156
|
return _allowedRealDirs;
|
|
152
157
|
}
|
|
@@ -190,22 +195,6 @@ function resolveSafePath(candidate, projectId) {
|
|
|
190
195
|
}
|
|
191
196
|
return null;
|
|
192
197
|
}
|
|
193
|
-
/** Resolve and validate a user-supplied video filename. Only bare .mp4 names in the output dir. */
|
|
194
|
-
function resolveSafeVideoPath(candidate, projectId) {
|
|
195
|
-
const base = candidate.split(/[/\\]/).pop() || candidate;
|
|
196
|
-
if (base !== candidate || base.includes("..") || base.startsWith(".")) {
|
|
197
|
-
return null;
|
|
198
|
-
}
|
|
199
|
-
if (!base.toLowerCase().endsWith(".mp4"))
|
|
200
|
-
return null;
|
|
201
|
-
const candidatePath = join(OUTPUT_DIR, projectId, base);
|
|
202
|
-
if (!existsSync(candidatePath))
|
|
203
|
-
return null;
|
|
204
|
-
const resolved = realpathSync(candidatePath);
|
|
205
|
-
if (isPathAllowed(resolved))
|
|
206
|
-
return resolved;
|
|
207
|
-
return null;
|
|
208
|
-
}
|
|
209
198
|
/**
|
|
210
199
|
* Resolve the `mlxgen` executable installed via `uv tool install --upgrade mlx-gen`.
|
|
211
200
|
* uv tool installs binaries into `~/.local/bin`; fall back to relying on PATH.
|
|
@@ -222,6 +211,63 @@ async function getMlxgenBin() {
|
|
|
222
211
|
}
|
|
223
212
|
return "mlxgen";
|
|
224
213
|
}
|
|
214
|
+
/** Resolve the `dots-tts` executable installed via `uv tool install dots-tts`. */
|
|
215
|
+
async function getDotsTtsBin() {
|
|
216
|
+
const candidates = [
|
|
217
|
+
join(homedir(), ".local", "bin", "dots-tts"),
|
|
218
|
+
"/opt/homebrew/bin/dots-tts",
|
|
219
|
+
"/usr/local/bin/dots-tts",
|
|
220
|
+
];
|
|
221
|
+
for (const p of candidates) {
|
|
222
|
+
if (existsSync(p))
|
|
223
|
+
return p;
|
|
224
|
+
}
|
|
225
|
+
return "dots-tts";
|
|
226
|
+
}
|
|
227
|
+
/** Resolve the `mlx_whisper` executable installed via `uv tool install mlx-whisper`. */
|
|
228
|
+
async function getMlxWhisperBin() {
|
|
229
|
+
const candidates = [
|
|
230
|
+
join(homedir(), ".local", "bin", "mlx_whisper"),
|
|
231
|
+
"/opt/homebrew/bin/mlx_whisper",
|
|
232
|
+
"/usr/local/bin/mlx_whisper",
|
|
233
|
+
];
|
|
234
|
+
for (const p of candidates) {
|
|
235
|
+
if (existsSync(p))
|
|
236
|
+
return p;
|
|
237
|
+
}
|
|
238
|
+
return "mlx_whisper";
|
|
239
|
+
}
|
|
240
|
+
/** The cloned dots-tts-mlx project directory. */
|
|
241
|
+
const DOTS_TTS_FOLDER = join(APP_DATA_DIR, "python-src", "dots-tts-mlx");
|
|
242
|
+
/** Base directory where dots-tts MLX weights live (inside the project folder). */
|
|
243
|
+
const DOTS_TTS_WEIGHTS_DIR = join(DOTS_TTS_FOLDER, "dots-tts-mlx-weights");
|
|
244
|
+
/**
|
|
245
|
+
* Resolve a dots-tts `--model` value. Accepts `./dots-tts-mlx-weights/mf-int4`
|
|
246
|
+
* (relative to the dots-tts-mlx folder), a bare variant (`mf-int4`), or an
|
|
247
|
+
* explicit path.
|
|
248
|
+
*/
|
|
249
|
+
function resolveDotsTtsModel(model) {
|
|
250
|
+
if (!model)
|
|
251
|
+
return join(DOTS_TTS_WEIGHTS_DIR, "mf-int4");
|
|
252
|
+
if (model.startsWith("./"))
|
|
253
|
+
return join(DOTS_TTS_FOLDER, model.slice(2));
|
|
254
|
+
if (model.includes("/"))
|
|
255
|
+
return model;
|
|
256
|
+
return join(DOTS_TTS_WEIGHTS_DIR, model);
|
|
257
|
+
}
|
|
258
|
+
/** True when the dots-tts `mf-int4` weights have been downloaded. */
|
|
259
|
+
function isDotsTtsModelDownloaded() {
|
|
260
|
+
return existsSync(join(DOTS_TTS_WEIGHTS_DIR, "mf-int4"));
|
|
261
|
+
}
|
|
262
|
+
/** Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel). */
|
|
263
|
+
async function getFfmpegBin() {
|
|
264
|
+
const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
|
|
265
|
+
for (const p of candidates) {
|
|
266
|
+
if (existsSync(p))
|
|
267
|
+
return p;
|
|
268
|
+
}
|
|
269
|
+
return "ffmpeg";
|
|
270
|
+
}
|
|
225
271
|
/** True when the `mlxgen` executable is installed (known paths or PATH). */
|
|
226
272
|
function isMlxgenInstalled() {
|
|
227
273
|
const candidates = [
|
|
@@ -296,18 +342,6 @@ function isMlxVlmInstalled() {
|
|
|
296
342
|
return false;
|
|
297
343
|
}
|
|
298
344
|
}
|
|
299
|
-
/**
|
|
300
|
-
* Resolve the ffmpeg binary installed via Homebrew (Apple Silicon then Intel).
|
|
301
|
-
* Falls back to relying on PATH when neither known location exists.
|
|
302
|
-
*/
|
|
303
|
-
async function getFfmpegBin() {
|
|
304
|
-
const candidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"];
|
|
305
|
-
for (const p of candidates) {
|
|
306
|
-
if (existsSync(p))
|
|
307
|
-
return p;
|
|
308
|
-
}
|
|
309
|
-
return "ffmpeg";
|
|
310
|
-
}
|
|
311
345
|
/** Fixed filename mlx_audio.tts.generate writes its output clip as. */
|
|
312
346
|
const TTS_OUTPUT_FILENAME = "audio_000.mp3";
|
|
313
347
|
/**
|
|
@@ -577,6 +611,76 @@ export async function generateSceneVideo(uvPath, projectId, scene, characters, o
|
|
|
577
611
|
url: `/api/files?path=${encodeURIComponent(videoPath)}`,
|
|
578
612
|
};
|
|
579
613
|
}
|
|
614
|
+
/**
|
|
615
|
+
* Generate a video from a project image via LTX-2.3. `imagePath` must be a bare
|
|
616
|
+
* filename previously uploaded/generated for this project. Used by the queue
|
|
617
|
+
* worker for the "Scene Video Generation" tab.
|
|
618
|
+
*/
|
|
619
|
+
export async function generateImageToVideo(uvPath, projectId, params, onLog) {
|
|
620
|
+
const { prompt, imagePath, mode } = params;
|
|
621
|
+
if (!isValidProjectId(projectId))
|
|
622
|
+
return { error: "Invalid project ID" };
|
|
623
|
+
if (!prompt || !prompt.trim())
|
|
624
|
+
return { error: "Prompt is required" };
|
|
625
|
+
if (!imagePath)
|
|
626
|
+
return { error: "Image path is required" };
|
|
627
|
+
const resolvedImage = resolveSafePath(imagePath, projectId);
|
|
628
|
+
if (!resolvedImage) {
|
|
629
|
+
return {
|
|
630
|
+
error: "Invalid image path. Provide a filename previously uploaded to this project.",
|
|
631
|
+
};
|
|
632
|
+
}
|
|
633
|
+
const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
|
|
634
|
+
if (!existsSync(ltxFolder)) {
|
|
635
|
+
return { error: "ltx-2-mlx not found. Run setup first." };
|
|
636
|
+
}
|
|
637
|
+
const projectOutputDir = resolveOutputDir(undefined, projectId);
|
|
638
|
+
if (!projectOutputDir)
|
|
639
|
+
return { error: "Invalid output directory." };
|
|
640
|
+
const outputFile = `video-${Date.now()}.mp4`;
|
|
641
|
+
const outputPath = join(projectOutputDir, outputFile);
|
|
642
|
+
const videoWidth = Number(params.width) || 480;
|
|
643
|
+
const videoHeight = Number(params.height) || 480;
|
|
644
|
+
const videoFrames = Number(params.frames) || 121;
|
|
645
|
+
const videoFps = Number(params.frameRate) || 24;
|
|
646
|
+
const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 30);
|
|
647
|
+
const stage2Steps = Math.max(1, Math.round(Number(params.stage2Steps)) || 3);
|
|
648
|
+
const result = await runCommand([
|
|
649
|
+
uvPath,
|
|
650
|
+
"run",
|
|
651
|
+
"ltx-2-mlx",
|
|
652
|
+
"generate",
|
|
653
|
+
"--model",
|
|
654
|
+
resolveLtxModel(params.model),
|
|
655
|
+
"--prompt",
|
|
656
|
+
prompt.trim(),
|
|
657
|
+
stageFlagFor(mode),
|
|
658
|
+
"--stage1-steps",
|
|
659
|
+
String(stage1Steps),
|
|
660
|
+
"--stage2-steps",
|
|
661
|
+
String(stage2Steps),
|
|
662
|
+
"--frames",
|
|
663
|
+
String(videoFrames),
|
|
664
|
+
"--width",
|
|
665
|
+
String(videoWidth),
|
|
666
|
+
"--height",
|
|
667
|
+
String(videoHeight),
|
|
668
|
+
"--frame-rate",
|
|
669
|
+
String(videoFps),
|
|
670
|
+
"--image",
|
|
671
|
+
resolvedImage,
|
|
672
|
+
"--output",
|
|
673
|
+
outputPath,
|
|
674
|
+
], { cwd: ltxFolder, onLog });
|
|
675
|
+
if (!result.success || !existsSync(outputPath)) {
|
|
676
|
+
return { error: result.output || "Video generation failed" };
|
|
677
|
+
}
|
|
678
|
+
backupFile(outputPath, projectId);
|
|
679
|
+
return {
|
|
680
|
+
filename: outputFile,
|
|
681
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
682
|
+
};
|
|
683
|
+
}
|
|
580
684
|
/** Normalize a value into a safe filesystem slug. */
|
|
581
685
|
function slugify(v) {
|
|
582
686
|
return String(v || "")
|
|
@@ -587,10 +691,11 @@ function slugify(v) {
|
|
|
587
691
|
* Generate a single scene image via fast-image-edit (FLUX.2 Klein), using the
|
|
588
692
|
* already-generated character/place images referenced by the scene's slugs.
|
|
589
693
|
*/
|
|
590
|
-
export async function generateSceneImage(projectId, scene, onLog) {
|
|
694
|
+
export async function generateSceneImage(projectId, scene, steps, onLog) {
|
|
591
695
|
const s = slugify(scene?.slug);
|
|
592
696
|
if (!s)
|
|
593
697
|
return { error: "Invalid scene slug" };
|
|
698
|
+
const stepCount = Math.max(1, Number(steps) || 6);
|
|
594
699
|
const outputDir = join(OUTPUT_DIR, projectId);
|
|
595
700
|
const mlxgen = await getMlxgenBin();
|
|
596
701
|
const refImages = [];
|
|
@@ -614,7 +719,7 @@ export async function generateSceneImage(projectId, scene, onLog) {
|
|
|
614
719
|
const fluxArgs = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
|
|
615
720
|
for (const p of refImages)
|
|
616
721
|
fluxArgs.push("--image", p);
|
|
617
|
-
fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps",
|
|
722
|
+
fluxArgs.push("--prompt", String(scene?.imagePrompt || ""), "--output", sceneImagePath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
|
|
618
723
|
const result = await runCommand(fluxArgs, { onLog });
|
|
619
724
|
if (!result.success || !existsSync(sceneImagePath)) {
|
|
620
725
|
return { error: result.output || `Failed to generate scene image ${s}` };
|
|
@@ -625,6 +730,439 @@ export async function generateSceneImage(projectId, scene, onLog) {
|
|
|
625
730
|
url: `/api/files?path=${encodeURIComponent(sceneImagePath)}`,
|
|
626
731
|
};
|
|
627
732
|
}
|
|
733
|
+
/**
|
|
734
|
+
* Upscale/refine a project image via mlxgen (SeedVR2). `imagePath` must be a
|
|
735
|
+
* bare filename previously uploaded/generated for this project. `resolution` is
|
|
736
|
+
* either "1x" (refine at native resolution) or "2048" (upscale to 2048px).
|
|
737
|
+
*/
|
|
738
|
+
export async function generateUpscale(projectId, imagePath, resolution, onLog) {
|
|
739
|
+
if (!isValidProjectId(projectId))
|
|
740
|
+
return { error: "Invalid project ID" };
|
|
741
|
+
if (!imagePath)
|
|
742
|
+
return { error: "Image path is required" };
|
|
743
|
+
const resolvedImage = resolveSafePath(imagePath, projectId);
|
|
744
|
+
if (!resolvedImage) {
|
|
745
|
+
return {
|
|
746
|
+
error: "Invalid image path. Provide a filename previously uploaded to this project.",
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
const target = /^(1x|\d+)$/.test(resolution) ? resolution : "1x";
|
|
750
|
+
const mlxgen = await getMlxgenBin();
|
|
751
|
+
const projectOutputDir = join(OUTPUT_DIR, projectId);
|
|
752
|
+
ensureDir(projectOutputDir);
|
|
753
|
+
const outputFile = `upscale-${target}-${Date.now()}.png`;
|
|
754
|
+
const outputPath = join(projectOutputDir, outputFile);
|
|
755
|
+
const result = await runCommand([
|
|
756
|
+
mlxgen,
|
|
757
|
+
"upscale",
|
|
758
|
+
"--model",
|
|
759
|
+
SEEDVR2_MODEL,
|
|
760
|
+
"--image-path",
|
|
761
|
+
resolvedImage,
|
|
762
|
+
"--resolution",
|
|
763
|
+
target,
|
|
764
|
+
"--seed",
|
|
765
|
+
"42",
|
|
766
|
+
"--mlx-cache-limit-gb",
|
|
767
|
+
"100",
|
|
768
|
+
"--output",
|
|
769
|
+
outputPath,
|
|
770
|
+
], { onLog });
|
|
771
|
+
if (!result.success || !existsSync(outputPath)) {
|
|
772
|
+
return { error: result.output || "Upscale failed" };
|
|
773
|
+
}
|
|
774
|
+
backupFile(outputPath, projectId);
|
|
775
|
+
return {
|
|
776
|
+
filename: outputFile,
|
|
777
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
/**
|
|
781
|
+
* Clone a reference voice and speak `text` via mlx_audio.tts.generate. `refAudioPath`
|
|
782
|
+
* must be a bare filename previously uploaded to this project. `quality` is "low"
|
|
783
|
+
* or "high" (maps to a TTS model). Output is saved under <output>/voices/.
|
|
784
|
+
*/
|
|
785
|
+
export async function generateVoiceClone(uvPath, projectId, text, refAudioPath, quality, onLog) {
|
|
786
|
+
if (!isValidProjectId(projectId))
|
|
787
|
+
return { error: "Invalid project ID" };
|
|
788
|
+
if (!refAudioPath)
|
|
789
|
+
return { error: "Reference audio is required" };
|
|
790
|
+
// Sanitize the transcript: strip control characters (incl. newlines) and
|
|
791
|
+
// collapse whitespace. spawn() runs with shell:false (array args), so shell
|
|
792
|
+
// metacharacters cannot execute, but this keeps `--text` a single well-formed
|
|
793
|
+
// argument and out of the terminal log.
|
|
794
|
+
const cleanText = text
|
|
795
|
+
.replace(/[\u0000-\u001f\u007f]/g, " ")
|
|
796
|
+
.replace(/\s+/g, " ")
|
|
797
|
+
.trim();
|
|
798
|
+
if (!cleanText)
|
|
799
|
+
return { error: "Text is required" };
|
|
800
|
+
const resolvedRef = resolveSafePath(refAudioPath, projectId);
|
|
801
|
+
if (!resolvedRef) {
|
|
802
|
+
return {
|
|
803
|
+
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
804
|
+
};
|
|
805
|
+
}
|
|
806
|
+
const model = quality === "low" ? TTS_MODELS.low : TTS_MODELS.high;
|
|
807
|
+
const projectOutputDir = resolveOutputDir(undefined, projectId);
|
|
808
|
+
if (!projectOutputDir)
|
|
809
|
+
return { error: "Invalid output directory." };
|
|
810
|
+
const voiceId = `voice-${Date.now()}`;
|
|
811
|
+
const voiceDir = join(projectOutputDir, "voices", voiceId);
|
|
812
|
+
ensureDir(voiceDir);
|
|
813
|
+
const result = await runCommand([
|
|
814
|
+
uvPath,
|
|
815
|
+
"run",
|
|
816
|
+
"mlx_audio.tts.generate",
|
|
817
|
+
"--model",
|
|
818
|
+
model,
|
|
819
|
+
"--text",
|
|
820
|
+
cleanText,
|
|
821
|
+
"--ref_audio",
|
|
822
|
+
resolvedRef,
|
|
823
|
+
"--output",
|
|
824
|
+
voiceDir,
|
|
825
|
+
"--audio_format",
|
|
826
|
+
"mp3",
|
|
827
|
+
"--play",
|
|
828
|
+
"--instruct",
|
|
829
|
+
"slow down speech",
|
|
830
|
+
], { cwd: voiceDir, onLog });
|
|
831
|
+
if (!result.success) {
|
|
832
|
+
return { error: result.output || "Voice generation failed" };
|
|
833
|
+
}
|
|
834
|
+
const path = resolveAudioFile(voiceDir);
|
|
835
|
+
if (!path) {
|
|
836
|
+
return { error: "TTS completed but no audio file was produced" };
|
|
837
|
+
}
|
|
838
|
+
const filename = path.split(sep).pop() || TTS_OUTPUT_FILENAME;
|
|
839
|
+
// Persist a per-voice metadata file so the generated-voice list can be rebuilt
|
|
840
|
+
// from the folder structure (each voice in its own folder with meta.json), with
|
|
841
|
+
// no central JSON index.
|
|
842
|
+
const meta = {
|
|
843
|
+
id: voiceId,
|
|
844
|
+
transcript: cleanText,
|
|
845
|
+
quality,
|
|
846
|
+
refAudioFilename: refAudioPath,
|
|
847
|
+
filename,
|
|
848
|
+
createdAt: new Date().toISOString(),
|
|
849
|
+
};
|
|
850
|
+
writeFileSync(join(voiceDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
851
|
+
return {
|
|
852
|
+
filename,
|
|
853
|
+
url: `/api/files?path=${encodeURIComponent(path)}`,
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
/** Find the newest video file (mp4/webm/mov) inside a directory. */
|
|
857
|
+
function findNewestVideo(dir) {
|
|
858
|
+
let entries = [];
|
|
859
|
+
try {
|
|
860
|
+
entries = readdirSync(dir);
|
|
861
|
+
}
|
|
862
|
+
catch {
|
|
863
|
+
return null;
|
|
864
|
+
}
|
|
865
|
+
let newest = null;
|
|
866
|
+
for (const name of entries) {
|
|
867
|
+
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
868
|
+
if (ext !== ".mp4" && ext !== ".webm" && ext !== ".mov")
|
|
869
|
+
continue;
|
|
870
|
+
const full = join(dir, name);
|
|
871
|
+
try {
|
|
872
|
+
const st = statSync(full);
|
|
873
|
+
if (!st.isFile())
|
|
874
|
+
continue;
|
|
875
|
+
if (!newest || st.mtimeMs > newest.mtime) {
|
|
876
|
+
newest = { path: full, mtime: st.mtimeMs };
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
catch {
|
|
880
|
+
// skip unreadable entries
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
return newest ? newest.path : null;
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Generate a video from an image + audio via ltx-2-mlx `a2v` (audio-to-video).
|
|
887
|
+
* `imagePath` and `audioPath` must be bare filenames previously uploaded/generated
|
|
888
|
+
* for this project. `stage1Steps` maps to `--stage1-steps` (15 default, 30 HD).
|
|
889
|
+
*/
|
|
890
|
+
export async function generateAudioToVideo(uvPath, projectId, params, onLog) {
|
|
891
|
+
if (!isValidProjectId(projectId))
|
|
892
|
+
return { error: "Invalid project ID" };
|
|
893
|
+
if (!params.imagePath)
|
|
894
|
+
return { error: "Image path is required" };
|
|
895
|
+
if (!params.audioPath)
|
|
896
|
+
return { error: "Audio path is required" };
|
|
897
|
+
const resolvedImage = resolveSafePath(params.imagePath, projectId);
|
|
898
|
+
if (!resolvedImage) {
|
|
899
|
+
return {
|
|
900
|
+
error: "Invalid image path. Provide a filename previously uploaded to this project.",
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
const resolvedAudio = resolveSafePath(params.audioPath, projectId);
|
|
904
|
+
if (!resolvedAudio) {
|
|
905
|
+
return {
|
|
906
|
+
error: "Invalid audio path. Provide a filename previously uploaded to this project.",
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
// Collapse whitespace so the prompt stays a single well-formed CLI argument.
|
|
910
|
+
const cleanPrompt = String(params.prompt || "")
|
|
911
|
+
.replace(/\s+/g, " ")
|
|
912
|
+
.trim() || "scene";
|
|
913
|
+
const stage1Steps = Math.max(1, Math.round(Number(params.stage1Steps)) || 15);
|
|
914
|
+
// 1 second = 24 frames, plus a terminal frame (24n + 1).
|
|
915
|
+
const frames = Math.max(1, Math.round(Number(params.frames)) || 25);
|
|
916
|
+
const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
|
|
917
|
+
if (!existsSync(ltxFolder)) {
|
|
918
|
+
return { error: "ltx-2-mlx not found. Run setup first." };
|
|
919
|
+
}
|
|
920
|
+
const outputDir = join(OUTPUT_DIR, projectId, `a2v-${Date.now()}`);
|
|
921
|
+
ensureDir(outputDir);
|
|
922
|
+
const result = await runCommand([
|
|
923
|
+
uvPath,
|
|
924
|
+
"run",
|
|
925
|
+
"ltx-2-mlx",
|
|
926
|
+
"a2v",
|
|
927
|
+
"--image",
|
|
928
|
+
resolvedImage,
|
|
929
|
+
"--audio",
|
|
930
|
+
resolvedAudio,
|
|
931
|
+
"--frame-rate",
|
|
932
|
+
"24",
|
|
933
|
+
"--frames",
|
|
934
|
+
String(frames),
|
|
935
|
+
"--output",
|
|
936
|
+
outputDir,
|
|
937
|
+
"--prompt",
|
|
938
|
+
cleanPrompt,
|
|
939
|
+
"--stage1-steps",
|
|
940
|
+
String(stage1Steps),
|
|
941
|
+
"--stage2-steps",
|
|
942
|
+
"3",
|
|
943
|
+
], { cwd: ltxFolder, onLog });
|
|
944
|
+
if (!result.success) {
|
|
945
|
+
return { error: result.output || "Audio-to-video generation failed" };
|
|
946
|
+
}
|
|
947
|
+
const videoPath = findNewestVideo(outputDir);
|
|
948
|
+
if (!videoPath) {
|
|
949
|
+
return { error: "a2v completed but no video file was produced" };
|
|
950
|
+
}
|
|
951
|
+
backupFile(videoPath, projectId);
|
|
952
|
+
return {
|
|
953
|
+
filename: videoPath.split(sep).pop() || "a2v.mp4",
|
|
954
|
+
url: `/api/files?path=${encodeURIComponent(videoPath)}`,
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
/**
|
|
958
|
+
* Clone a reference voice via dots-tts. Output is written to
|
|
959
|
+
* <output>/<projectId>/dots-tts/ as `<prefix>_000.wav`.
|
|
960
|
+
*/
|
|
961
|
+
export async function generateAdvancedVoiceClone(projectId, params, onLog) {
|
|
962
|
+
if (!isValidProjectId(projectId))
|
|
963
|
+
return { error: "Invalid project ID" };
|
|
964
|
+
const cleanText = String(params.text || "")
|
|
965
|
+
.replace(/\s+/g, " ")
|
|
966
|
+
.trim();
|
|
967
|
+
if (!cleanText)
|
|
968
|
+
return { error: "Text is required" };
|
|
969
|
+
if (!params.refAudioPath)
|
|
970
|
+
return { error: "Reference audio is required" };
|
|
971
|
+
const resolvedRef = resolveSafePath(params.refAudioPath, projectId);
|
|
972
|
+
if (!resolvedRef) {
|
|
973
|
+
return {
|
|
974
|
+
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
975
|
+
};
|
|
976
|
+
}
|
|
977
|
+
const language = String(params.language || "YUE").trim() || "YUE";
|
|
978
|
+
const prefix = String(params.outPrefix || "voice")
|
|
979
|
+
.replace(/[^a-zA-Z0-9_-]/g, "_")
|
|
980
|
+
.slice(0, 64) || "voice";
|
|
981
|
+
const model = resolveDotsTtsModel(String(params.model || ""));
|
|
982
|
+
// dots-tts and mlx_whisper both expect a WAV reference — convert mp3/other
|
|
983
|
+
// formats to 16-bit PCM WAV first.
|
|
984
|
+
let refAudio = resolvedRef;
|
|
985
|
+
if (!resolvedRef.toLowerCase().endsWith(".wav")) {
|
|
986
|
+
const tempDir = join(TEMP_DIR, String(projectId));
|
|
987
|
+
ensureDir(tempDir);
|
|
988
|
+
const convertedRef = join(tempDir, `avc-ref-${Date.now()}.wav`);
|
|
989
|
+
const ffmpegBin = await getFfmpegBin();
|
|
990
|
+
const conv = await runCommand([
|
|
991
|
+
ffmpegBin,
|
|
992
|
+
"-y",
|
|
993
|
+
"-i",
|
|
994
|
+
resolvedRef,
|
|
995
|
+
"-c:a",
|
|
996
|
+
"pcm_s16le",
|
|
997
|
+
"-ar",
|
|
998
|
+
"44100",
|
|
999
|
+
"-ac",
|
|
1000
|
+
"2",
|
|
1001
|
+
convertedRef,
|
|
1002
|
+
], { onLog });
|
|
1003
|
+
if (!conv.success || !existsSync(convertedRef)) {
|
|
1004
|
+
return { error: "Failed to convert reference audio to WAV" };
|
|
1005
|
+
}
|
|
1006
|
+
refAudio = convertedRef;
|
|
1007
|
+
}
|
|
1008
|
+
// dots-tts requires `--ref-text` (the reference audio's transcript). Transcribe
|
|
1009
|
+
// the WAV reference with mlx_whisper to supply it.
|
|
1010
|
+
const whisperBin = await getMlxWhisperBin();
|
|
1011
|
+
const transcribeDir = join(TEMP_DIR, String(projectId));
|
|
1012
|
+
ensureDir(transcribeDir);
|
|
1013
|
+
const whisperResult = await runCommand([whisperBin, refAudio, "--output-dir", transcribeDir], { onLog });
|
|
1014
|
+
const refTxtPath = join(transcribeDir, (refAudio.split(sep).pop() || "ref").replace(/\.[^.]+$/, "") + ".txt");
|
|
1015
|
+
const refText = whisperResult.success && existsSync(refTxtPath)
|
|
1016
|
+
? readFileSync(refTxtPath, "utf-8").replace(/\s+/g, " ").trim()
|
|
1017
|
+
: "";
|
|
1018
|
+
if (!refText) {
|
|
1019
|
+
return { error: "Failed to transcribe reference audio (mlx_whisper)" };
|
|
1020
|
+
}
|
|
1021
|
+
const dotsTtsBin = await getDotsTtsBin();
|
|
1022
|
+
// Mirror the voice-clone tab's timestamped-folder layout so outputs never
|
|
1023
|
+
// overwrite each other.
|
|
1024
|
+
const voiceId = `voice-${Date.now()}`;
|
|
1025
|
+
const outDir = join(OUTPUT_DIR, projectId, "dots-tts", voiceId);
|
|
1026
|
+
ensureDir(outDir);
|
|
1027
|
+
console.log("debug-info:", [
|
|
1028
|
+
dotsTtsBin,
|
|
1029
|
+
"--model",
|
|
1030
|
+
model,
|
|
1031
|
+
"--text",
|
|
1032
|
+
cleanText || "please provide text",
|
|
1033
|
+
"--ref-audio",
|
|
1034
|
+
refAudio,
|
|
1035
|
+
"--ref-text",
|
|
1036
|
+
refText,
|
|
1037
|
+
"--language",
|
|
1038
|
+
language,
|
|
1039
|
+
"--out-path",
|
|
1040
|
+
outDir,
|
|
1041
|
+
"--out-prefix",
|
|
1042
|
+
prefix,
|
|
1043
|
+
]);
|
|
1044
|
+
const result = await runCommand([
|
|
1045
|
+
dotsTtsBin,
|
|
1046
|
+
"--model",
|
|
1047
|
+
model,
|
|
1048
|
+
"--text",
|
|
1049
|
+
cleanText || "please provide text",
|
|
1050
|
+
"--ref-audio",
|
|
1051
|
+
refAudio,
|
|
1052
|
+
"--ref-text",
|
|
1053
|
+
refText,
|
|
1054
|
+
"--language",
|
|
1055
|
+
language,
|
|
1056
|
+
"--out-path",
|
|
1057
|
+
outDir,
|
|
1058
|
+
"--out-prefix",
|
|
1059
|
+
prefix,
|
|
1060
|
+
], { cwd: dirname(dotsTtsBin), onLog });
|
|
1061
|
+
if (!result.success) {
|
|
1062
|
+
return { error: result.output || "Advanced voice clone failed" };
|
|
1063
|
+
}
|
|
1064
|
+
const outputFile = `${prefix}_000.wav`;
|
|
1065
|
+
const outputPath = join(outDir, outputFile);
|
|
1066
|
+
if (!existsSync(outputPath)) {
|
|
1067
|
+
return { error: `Expected output ${outputFile} was not produced` };
|
|
1068
|
+
}
|
|
1069
|
+
// Mirror the voice-clone tab's on-disk conventions: a timestamped backup copy
|
|
1070
|
+
// plus a per-voice meta.json so the output can be listed without a central index.
|
|
1071
|
+
backupFile(outputPath, projectId);
|
|
1072
|
+
const meta = {
|
|
1073
|
+
id: voiceId,
|
|
1074
|
+
transcript: cleanText,
|
|
1075
|
+
refText,
|
|
1076
|
+
language,
|
|
1077
|
+
model,
|
|
1078
|
+
refAudioFilename: params.refAudioPath,
|
|
1079
|
+
outPrefix: prefix,
|
|
1080
|
+
filename: outputFile,
|
|
1081
|
+
createdAt: new Date().toISOString(),
|
|
1082
|
+
};
|
|
1083
|
+
writeFileSync(join(outDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
1084
|
+
return {
|
|
1085
|
+
filename: outputFile,
|
|
1086
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
/**
|
|
1090
|
+
* Generate a composite image via fast-image-edit (FLUX.2 Klein). `images` are
|
|
1091
|
+
* base64 data URLs that are decoded into temp files and passed to the model as
|
|
1092
|
+
* separate `--image` inputs. Used by the generation queue worker.
|
|
1093
|
+
*/
|
|
1094
|
+
export async function generateFastImageEditImage(projectId, prompt, images, steps, upscaleResolution, onLog) {
|
|
1095
|
+
if (!isValidProjectId(projectId))
|
|
1096
|
+
return { error: "Invalid project ID" };
|
|
1097
|
+
if (!prompt || !prompt.trim())
|
|
1098
|
+
return { error: "Prompt is required" };
|
|
1099
|
+
if (!Array.isArray(images) || images.length === 0) {
|
|
1100
|
+
return { error: "At least one reference image is required" };
|
|
1101
|
+
}
|
|
1102
|
+
const stepCount = Math.max(1, Number(steps) || 4);
|
|
1103
|
+
// Decode each base64 reference image into a temp workspace file so the
|
|
1104
|
+
// FLUX model receives them as separate `--image` inputs.
|
|
1105
|
+
const tempDir = join(TEMP_DIR, String(projectId));
|
|
1106
|
+
ensureDir(tempDir);
|
|
1107
|
+
const tempImagePaths = [];
|
|
1108
|
+
try {
|
|
1109
|
+
images.forEach((image, i) => {
|
|
1110
|
+
const base64 = String(image).replace(/^data:image\/\w+;base64,/, "");
|
|
1111
|
+
const buffer = Buffer.from(base64, "base64");
|
|
1112
|
+
const path = join(tempDir, `flux-ref-${Date.now()}-${i}.png`);
|
|
1113
|
+
writeFileSync(path, buffer);
|
|
1114
|
+
tempImagePaths.push(path);
|
|
1115
|
+
});
|
|
1116
|
+
}
|
|
1117
|
+
catch {
|
|
1118
|
+
return { error: "Invalid reference image data" };
|
|
1119
|
+
}
|
|
1120
|
+
try {
|
|
1121
|
+
const mlxgen = await getMlxgenBin();
|
|
1122
|
+
const projectOutputDir = join(OUTPUT_DIR, projectId);
|
|
1123
|
+
ensureDir(projectOutputDir);
|
|
1124
|
+
const outputFile = `flux-edit-${Date.now()}.png`;
|
|
1125
|
+
const outputPath = join(projectOutputDir, outputFile);
|
|
1126
|
+
const args = [mlxgen, "generate", "--model", FLUX_KLEIN_MODEL];
|
|
1127
|
+
for (const path of tempImagePaths)
|
|
1128
|
+
args.push("--image", path);
|
|
1129
|
+
args.push("--prompt", prompt.trim(), "--output", outputPath, "--mlx-cache-limit-gb", "20", "--steps", String(stepCount), "--seed", "42", "--width", "1024", "--height", "1024");
|
|
1130
|
+
const result = await runCommand(args, { onLog });
|
|
1131
|
+
if (!result.success || !existsSync(outputPath)) {
|
|
1132
|
+
return { error: result.output || "Fast image edit failed" };
|
|
1133
|
+
}
|
|
1134
|
+
backupFile(outputPath, projectId);
|
|
1135
|
+
// Optionally upscale the generated result (1x / 1500px / 2000px).
|
|
1136
|
+
if (upscaleResolution && upscaleResolution !== "none") {
|
|
1137
|
+
if (onLog)
|
|
1138
|
+
onLog(`Upscaling result (${upscaleResolution})…\n`);
|
|
1139
|
+
const upscaled = await generateUpscale(projectId, outputFile, upscaleResolution, onLog);
|
|
1140
|
+
if ("error" in upscaled)
|
|
1141
|
+
return { error: upscaled.error };
|
|
1142
|
+
return upscaled;
|
|
1143
|
+
}
|
|
1144
|
+
return {
|
|
1145
|
+
filename: outputFile,
|
|
1146
|
+
url: `/api/files?path=${encodeURIComponent(outputPath)}`,
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
finally {
|
|
1150
|
+
for (const path of tempImagePaths) {
|
|
1151
|
+
try {
|
|
1152
|
+
unlinkSync(path);
|
|
1153
|
+
}
|
|
1154
|
+
catch {
|
|
1155
|
+
// already removed
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
try {
|
|
1159
|
+
rmSync(tempDir, { force: true });
|
|
1160
|
+
}
|
|
1161
|
+
catch {
|
|
1162
|
+
// ignore cleanup failures
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
628
1166
|
// ========== Routes ==========
|
|
629
1167
|
export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
630
1168
|
// ========== Upload ==========
|
|
@@ -909,6 +1447,60 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
909
1447
|
raw.sort((a, b) => b.birthtime - a.birthtime);
|
|
910
1448
|
res.json(raw.map(({ filename, url }) => ({ filename, url })));
|
|
911
1449
|
});
|
|
1450
|
+
// List previously generated voice clones by scanning the voices folder. Each
|
|
1451
|
+
// voice lives in its own subfolder with a meta.json (no central JSON index).
|
|
1452
|
+
app.get("/api/projects/:id/voices", (req, res) => {
|
|
1453
|
+
const { id } = req.params;
|
|
1454
|
+
if (!isValidProjectId(id)) {
|
|
1455
|
+
res.status(400).json({ error: "Invalid project ID" });
|
|
1456
|
+
return;
|
|
1457
|
+
}
|
|
1458
|
+
const voicesRoot = join(OUTPUT_DIR, id, "voices");
|
|
1459
|
+
const results = [];
|
|
1460
|
+
if (existsSync(voicesRoot)) {
|
|
1461
|
+
let names = [];
|
|
1462
|
+
try {
|
|
1463
|
+
names = readdirSync(voicesRoot);
|
|
1464
|
+
}
|
|
1465
|
+
catch {
|
|
1466
|
+
names = [];
|
|
1467
|
+
}
|
|
1468
|
+
for (const name of names) {
|
|
1469
|
+
const voiceDir = join(voicesRoot, name);
|
|
1470
|
+
let isDir = false;
|
|
1471
|
+
try {
|
|
1472
|
+
isDir = statSync(voiceDir).isDirectory();
|
|
1473
|
+
}
|
|
1474
|
+
catch {
|
|
1475
|
+
isDir = false;
|
|
1476
|
+
}
|
|
1477
|
+
if (!isDir)
|
|
1478
|
+
continue;
|
|
1479
|
+
let meta = null;
|
|
1480
|
+
try {
|
|
1481
|
+
meta = JSON.parse(readFileSync(join(voiceDir, "meta.json"), "utf-8"));
|
|
1482
|
+
}
|
|
1483
|
+
catch {
|
|
1484
|
+
continue; // folder without a meta.json — skip
|
|
1485
|
+
}
|
|
1486
|
+
const filename = String(meta?.filename || TTS_OUTPUT_FILENAME);
|
|
1487
|
+
const audioPath = join(voiceDir, filename);
|
|
1488
|
+
if (!existsSync(audioPath))
|
|
1489
|
+
continue;
|
|
1490
|
+
results.push({
|
|
1491
|
+
id: name,
|
|
1492
|
+
transcript: String(meta?.transcript ?? ""),
|
|
1493
|
+
quality: String(meta?.quality ?? "high"),
|
|
1494
|
+
refAudioFilename: meta?.refAudioFilename ?? null,
|
|
1495
|
+
filename,
|
|
1496
|
+
createdAt: meta?.createdAt ?? null,
|
|
1497
|
+
url: `/api/files?path=${encodeURIComponent(audioPath)}`,
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
results.sort((a, b) => String(b.createdAt ?? "").localeCompare(String(a.createdAt ?? "")));
|
|
1502
|
+
res.json(results);
|
|
1503
|
+
});
|
|
912
1504
|
// ========== Render: Text-to-Image ==========
|
|
913
1505
|
app.post("/api/render/text-to-image", async (req, res) => {
|
|
914
1506
|
const { prompt, projectId, width = 512, height = 512, device = "mps", } = req.body || {};
|
|
@@ -1002,7 +1594,7 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1002
1594
|
});
|
|
1003
1595
|
// ========== Render: Image-to-Video ==========
|
|
1004
1596
|
app.post("/api/render/image-to-video", async (req, res) => {
|
|
1005
|
-
const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", } = req.body || {};
|
|
1597
|
+
const { prompt, imagePath, projectId, outputDir, width = 480, height = 480, frames = 121, frameRate = 24, mode = "distilled", stage1Steps, stage2Steps, model, } = req.body || {};
|
|
1006
1598
|
if (!prompt) {
|
|
1007
1599
|
res.status(400).json({ error: "Prompt is required" });
|
|
1008
1600
|
return;
|
|
@@ -1058,6 +1650,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1058
1650
|
const videoHeight = Number(height) || 480;
|
|
1059
1651
|
const videoFrames = Number(frames) || 121;
|
|
1060
1652
|
const videoFps = Number(frameRate) || 24;
|
|
1653
|
+
const stage1 = Math.max(1, Math.round(Number(stage1Steps)) || 30);
|
|
1654
|
+
const stage2 = Math.max(1, Math.round(Number(stage2Steps)) || 3);
|
|
1061
1655
|
send("progress", {
|
|
1062
1656
|
status: "starting",
|
|
1063
1657
|
label: "Generating video...",
|
|
@@ -1075,10 +1669,14 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1075
1669
|
"ltx-2-mlx",
|
|
1076
1670
|
"generate",
|
|
1077
1671
|
"--model",
|
|
1078
|
-
|
|
1672
|
+
resolveLtxModel(model),
|
|
1079
1673
|
"--prompt",
|
|
1080
1674
|
prompt,
|
|
1081
1675
|
stageFlagFor(mode),
|
|
1676
|
+
"--stage1-steps",
|
|
1677
|
+
String(stage1),
|
|
1678
|
+
"--stage2-steps",
|
|
1679
|
+
String(stage2),
|
|
1082
1680
|
// "--low-ram",
|
|
1083
1681
|
"--frames",
|
|
1084
1682
|
String(videoFrames),
|
|
@@ -1674,252 +2272,19 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
1674
2272
|
res.end();
|
|
1675
2273
|
}
|
|
1676
2274
|
});
|
|
1677
|
-
// ==========
|
|
1678
|
-
app.
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
return;
|
|
1683
|
-
}
|
|
1684
|
-
if (!videoPath) {
|
|
1685
|
-
res.status(400).json({ error: "Video path is required" });
|
|
1686
|
-
return;
|
|
2275
|
+
// ========== MLX-Audio: Status ==========
|
|
2276
|
+
app.get("/api/mlxaudio/status", async (_req, res) => {
|
|
2277
|
+
// Never throw on a missing uv — installation state is based on the folder.
|
|
2278
|
+
try {
|
|
2279
|
+
await getUvPath();
|
|
1687
2280
|
}
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
return;
|
|
1691
|
-
}
|
|
1692
|
-
// Resolve video path — only bare .mp4 filenames in this project's output dir
|
|
1693
|
-
const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
|
|
1694
|
-
if (!resolvedVideo) {
|
|
1695
|
-
res.status(400).json({
|
|
1696
|
-
error: "Invalid video path. Provide a filename previously generated in this project.",
|
|
1697
|
-
});
|
|
1698
|
-
return;
|
|
1699
|
-
}
|
|
1700
|
-
// SSE headers
|
|
1701
|
-
res.writeHead(200, {
|
|
1702
|
-
"Content-Type": "text/event-stream",
|
|
1703
|
-
"Cache-Control": "no-cache",
|
|
1704
|
-
Connection: "keep-alive",
|
|
1705
|
-
"X-Accel-Buffering": "no",
|
|
1706
|
-
});
|
|
1707
|
-
const send = (event, data) => {
|
|
1708
|
-
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
1709
|
-
};
|
|
1710
|
-
try {
|
|
1711
|
-
const ltxFolder = join(PYTHON_DIR, "ltx-2-mlx");
|
|
1712
|
-
if (!existsSync(ltxFolder)) {
|
|
1713
|
-
send("error", { error: "ltx-2-mlx not found. Run setup first." });
|
|
1714
|
-
res.end();
|
|
1715
|
-
return;
|
|
1716
|
-
}
|
|
1717
|
-
const uvPath = await getUvPath();
|
|
1718
|
-
const projectOutputDir = join(OUTPUT_DIR, projectId);
|
|
1719
|
-
ensureDir(projectOutputDir);
|
|
1720
|
-
const outputFile = `extended-${Date.now()}.mp4`;
|
|
1721
|
-
const outputPath = join(projectOutputDir, outputFile);
|
|
1722
|
-
const framesToAdd = Number(extendFrames) || 2;
|
|
1723
|
-
send("progress", {
|
|
1724
|
-
status: "starting",
|
|
1725
|
-
label: "Extending video...",
|
|
1726
|
-
inputFile: resolvedVideo,
|
|
1727
|
-
outputFile,
|
|
1728
|
-
settings: { extendFrames: framesToAdd },
|
|
1729
|
-
});
|
|
1730
|
-
const proc = spawn([
|
|
1731
|
-
uvPath,
|
|
1732
|
-
"run",
|
|
1733
|
-
"ltx-2-mlx",
|
|
1734
|
-
"extend",
|
|
1735
|
-
"--model",
|
|
1736
|
-
"dgrauet/ltx-2.3-mlx-q8",
|
|
1737
|
-
"--prompt",
|
|
1738
|
-
prompt,
|
|
1739
|
-
"--video",
|
|
1740
|
-
resolvedVideo,
|
|
1741
|
-
"--extend-frames",
|
|
1742
|
-
String(framesToAdd),
|
|
1743
|
-
"--output",
|
|
1744
|
-
outputPath,
|
|
1745
|
-
], {
|
|
1746
|
-
env: process.env,
|
|
1747
|
-
cwd: ltxFolder,
|
|
1748
|
-
stdout: "pipe",
|
|
1749
|
-
stderr: "pipe",
|
|
1750
|
-
});
|
|
1751
|
-
activeProc = proc;
|
|
1752
|
-
// Stream stdout/stderr concurrently
|
|
1753
|
-
const stdoutPromise = streamToSSE(proc.stdout, "Extend", send);
|
|
1754
|
-
const stderrText = await streamToSSE(proc.stderr, "Extend", send);
|
|
1755
|
-
await stdoutPromise;
|
|
1756
|
-
const exitCode = await proc.exited;
|
|
1757
|
-
const success = exitCode === 0 && existsSync(outputPath);
|
|
1758
|
-
if (success) {
|
|
1759
|
-
send("complete", {
|
|
1760
|
-
success: true,
|
|
1761
|
-
path: outputPath,
|
|
1762
|
-
filename: outputFile,
|
|
1763
|
-
});
|
|
1764
|
-
}
|
|
1765
|
-
else {
|
|
1766
|
-
send("error", {
|
|
1767
|
-
error: stderrText || `Process exited with code ${exitCode}`,
|
|
1768
|
-
exitCode,
|
|
1769
|
-
});
|
|
1770
|
-
}
|
|
1771
|
-
}
|
|
1772
|
-
catch (e) {
|
|
1773
|
-
send("error", { error: String(e) });
|
|
1774
|
-
}
|
|
1775
|
-
finally {
|
|
1776
|
-
activeProc = null;
|
|
1777
|
-
res.end();
|
|
1778
|
-
}
|
|
1779
|
-
});
|
|
1780
|
-
// ========== MLX-Audio: Status ==========
|
|
1781
|
-
app.get("/api/mlxaudio/status", async (_req, res) => {
|
|
1782
|
-
// Never throw on a missing uv — installation state is based on the folder.
|
|
1783
|
-
try {
|
|
1784
|
-
await getUvPath();
|
|
1785
|
-
}
|
|
1786
|
-
catch {
|
|
1787
|
-
// uv not found; installed is still reported from the folder below.
|
|
2281
|
+
catch {
|
|
2282
|
+
// uv not found; installed is still reported from the folder below.
|
|
1788
2283
|
}
|
|
1789
2284
|
res.json({
|
|
1790
2285
|
installed: existsSync(join(PYTHON_DIR, "mlx-audio")),
|
|
1791
2286
|
});
|
|
1792
2287
|
});
|
|
1793
|
-
// ========== Render: Text-to-Speech (mlx-audio) ==========
|
|
1794
|
-
app.post("/api/render/tts", async (req, res) => {
|
|
1795
|
-
const { text, refAudioPath, projectId, outputDir, quality, voiceId } = req.body || {};
|
|
1796
|
-
if (!text || typeof text !== "string" || !text.trim()) {
|
|
1797
|
-
res.status(400).json({ error: "Text is required" });
|
|
1798
|
-
return;
|
|
1799
|
-
}
|
|
1800
|
-
if (!refAudioPath) {
|
|
1801
|
-
res.status(400).json({ error: "Reference audio is required" });
|
|
1802
|
-
return;
|
|
1803
|
-
}
|
|
1804
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
1805
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
1806
|
-
return;
|
|
1807
|
-
}
|
|
1808
|
-
if (quality !== "low" && quality !== "high") {
|
|
1809
|
-
res.status(400).json({ error: "Quality must be 'low' or 'high'" });
|
|
1810
|
-
return;
|
|
1811
|
-
}
|
|
1812
|
-
// Resolve reference audio — only bare filenames in this project's dirs
|
|
1813
|
-
const resolvedRef = resolveSafePath(refAudioPath, projectId);
|
|
1814
|
-
if (!resolvedRef) {
|
|
1815
|
-
res.status(400).json({
|
|
1816
|
-
error: "Invalid reference audio path. Provide a filename previously uploaded to this project.",
|
|
1817
|
-
});
|
|
1818
|
-
return;
|
|
1819
|
-
}
|
|
1820
|
-
// SSE headers
|
|
1821
|
-
res.writeHead(200, {
|
|
1822
|
-
"Content-Type": "text/event-stream",
|
|
1823
|
-
"Cache-Control": "no-cache",
|
|
1824
|
-
Connection: "keep-alive",
|
|
1825
|
-
"X-Accel-Buffering": "no",
|
|
1826
|
-
});
|
|
1827
|
-
const send = (event, data) => {
|
|
1828
|
-
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
1829
|
-
};
|
|
1830
|
-
try {
|
|
1831
|
-
const uvPath = await getUvPath();
|
|
1832
|
-
const projectOutputDir = resolveOutputDir(outputDir, projectId);
|
|
1833
|
-
if (!projectOutputDir) {
|
|
1834
|
-
send("error", { error: "Invalid output directory." });
|
|
1835
|
-
res.end();
|
|
1836
|
-
return;
|
|
1837
|
-
}
|
|
1838
|
-
// TTS audio is saved under <projectOutputDir>/voices/<voiceId>/ so each
|
|
1839
|
-
// row's voiceover stays isolated in its own folder. Sanitize voiceId to
|
|
1840
|
-
// alphanumerics/_/- so it can never escape the voices directory.
|
|
1841
|
-
const safeVoiceId = String(voiceId ?? `voice-${Date.now()}`)
|
|
1842
|
-
.replace(/[^a-zA-Z0-9_-]/g, "_")
|
|
1843
|
-
.slice(0, 64);
|
|
1844
|
-
const voiceDir = join(projectOutputDir, "voices", safeVoiceId);
|
|
1845
|
-
ensureDir(voiceDir);
|
|
1846
|
-
// Regenerating a row must overwrite the previous clip: delete any
|
|
1847
|
-
// existing output so the generator writes a fresh audio_000.mp3.
|
|
1848
|
-
const prevAudio = join(voiceDir, TTS_OUTPUT_FILENAME);
|
|
1849
|
-
if (existsSync(prevAudio)) {
|
|
1850
|
-
try {
|
|
1851
|
-
unlinkSync(prevAudio);
|
|
1852
|
-
}
|
|
1853
|
-
catch {
|
|
1854
|
-
// Ignore — the generator will overwrite the file regardless.
|
|
1855
|
-
}
|
|
1856
|
-
}
|
|
1857
|
-
send("progress", {
|
|
1858
|
-
status: "starting",
|
|
1859
|
-
label: "Generating speech...",
|
|
1860
|
-
model: TTS_MODELS[quality],
|
|
1861
|
-
});
|
|
1862
|
-
// No --play flag: this is a silent server-side batch generation. The mp3
|
|
1863
|
-
// lands in <projectOutputDir>/voices/<voiceId>/, which is servable via
|
|
1864
|
-
// /api/files and resolvable via resolveSafePath for the mux step.
|
|
1865
|
-
const proc = spawn([
|
|
1866
|
-
uvPath,
|
|
1867
|
-
"run",
|
|
1868
|
-
"mlx_audio.tts.generate",
|
|
1869
|
-
"--model",
|
|
1870
|
-
TTS_MODELS[quality],
|
|
1871
|
-
"--text",
|
|
1872
|
-
text,
|
|
1873
|
-
"--ref_audio",
|
|
1874
|
-
resolvedRef,
|
|
1875
|
-
"--output",
|
|
1876
|
-
voiceDir,
|
|
1877
|
-
"--audio_format",
|
|
1878
|
-
"mp3",
|
|
1879
|
-
"--play",
|
|
1880
|
-
"--instruct",
|
|
1881
|
-
"slow down speech",
|
|
1882
|
-
], {
|
|
1883
|
-
env: process.env,
|
|
1884
|
-
cwd: voiceDir,
|
|
1885
|
-
stdout: "pipe",
|
|
1886
|
-
stderr: "pipe",
|
|
1887
|
-
});
|
|
1888
|
-
activeProc = proc;
|
|
1889
|
-
const stdoutPromise = streamToSSE(proc.stdout, "TTS", send);
|
|
1890
|
-
const stderrText = await streamToSSE(proc.stderr, "TTS", send);
|
|
1891
|
-
await stdoutPromise;
|
|
1892
|
-
const exitCode = await proc.exited;
|
|
1893
|
-
if (exitCode === 0) {
|
|
1894
|
-
const path = resolveAudioFile(voiceDir);
|
|
1895
|
-
if (path) {
|
|
1896
|
-
send("complete", {
|
|
1897
|
-
success: true,
|
|
1898
|
-
path,
|
|
1899
|
-
filename: path.split(sep).pop(),
|
|
1900
|
-
});
|
|
1901
|
-
}
|
|
1902
|
-
else {
|
|
1903
|
-
send("error", {
|
|
1904
|
-
error: "TTS completed but no audio file was produced",
|
|
1905
|
-
});
|
|
1906
|
-
}
|
|
1907
|
-
}
|
|
1908
|
-
else {
|
|
1909
|
-
send("error", {
|
|
1910
|
-
error: stderrText || `Process exited with code ${exitCode}`,
|
|
1911
|
-
exitCode,
|
|
1912
|
-
});
|
|
1913
|
-
}
|
|
1914
|
-
}
|
|
1915
|
-
catch (e) {
|
|
1916
|
-
send("error", { error: String(e) });
|
|
1917
|
-
}
|
|
1918
|
-
finally {
|
|
1919
|
-
activeProc = null;
|
|
1920
|
-
res.end();
|
|
1921
|
-
}
|
|
1922
|
-
});
|
|
1923
2288
|
// ========== Render: Voice Chat (TTS with reference voice) ==========
|
|
1924
2289
|
app.post("/api/render/voice-chat", async (req, res) => {
|
|
1925
2290
|
const { text, refAudioPath, projectId } = req.body || {};
|
|
@@ -2029,38 +2394,17 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2029
2394
|
res.end();
|
|
2030
2395
|
}
|
|
2031
2396
|
});
|
|
2032
|
-
// ==========
|
|
2033
|
-
app.
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
2044
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
2045
|
-
return;
|
|
2046
|
-
}
|
|
2047
|
-
// Resolve video path — only bare .mp4 filenames in this project's output dir
|
|
2048
|
-
const resolvedVideo = resolveSafeVideoPath(videoPath, projectId);
|
|
2049
|
-
if (!resolvedVideo) {
|
|
2050
|
-
res.status(400).json({
|
|
2051
|
-
error: "Invalid video path. Provide a filename previously generated in this project.",
|
|
2052
|
-
});
|
|
2053
|
-
return;
|
|
2054
|
-
}
|
|
2055
|
-
// Resolve audio path — only bare filenames in this project's dirs
|
|
2056
|
-
const resolvedAudio = resolveSafePath(audioPath, projectId);
|
|
2057
|
-
if (!resolvedAudio) {
|
|
2058
|
-
res.status(400).json({
|
|
2059
|
-
error: "Invalid audio path. Provide a filename previously generated or uploaded to this project.",
|
|
2060
|
-
});
|
|
2061
|
-
return;
|
|
2062
|
-
}
|
|
2063
|
-
// SSE headers
|
|
2397
|
+
// ========== MLX-Gen: Status ==========
|
|
2398
|
+
app.get("/api/mlxgen/status", (_req, res) => {
|
|
2399
|
+
res.json({
|
|
2400
|
+
installed: isMlxgenInstalled(),
|
|
2401
|
+
zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
|
|
2402
|
+
fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
|
|
2403
|
+
seedvr2Downloaded: isModelDownloaded(SEEDVR2_MODEL),
|
|
2404
|
+
});
|
|
2405
|
+
});
|
|
2406
|
+
// ========== MLX-Gen: Install ==========
|
|
2407
|
+
app.post("/api/mlxgen/install", async (_req, res) => {
|
|
2064
2408
|
res.writeHead(200, {
|
|
2065
2409
|
"Content-Type": "text/event-stream",
|
|
2066
2410
|
"Cache-Control": "no-cache",
|
|
@@ -2071,54 +2415,22 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2071
2415
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2072
2416
|
};
|
|
2073
2417
|
try {
|
|
2074
|
-
const
|
|
2075
|
-
const projectOutputDir = resolveOutputDir(outputDir, projectId);
|
|
2076
|
-
if (!projectOutputDir) {
|
|
2077
|
-
send("error", { error: "Invalid output directory." });
|
|
2078
|
-
res.end();
|
|
2079
|
-
return;
|
|
2080
|
-
}
|
|
2081
|
-
const finalFile = `voice-${Date.now()}.mp4`;
|
|
2082
|
-
const finalPath = join(projectOutputDir, finalFile);
|
|
2418
|
+
const uvPath = await getUvPath();
|
|
2083
2419
|
send("progress", {
|
|
2084
2420
|
status: "starting",
|
|
2085
|
-
label: "
|
|
2086
|
-
outputFile: finalFile,
|
|
2421
|
+
label: "Installing mlx-gen...",
|
|
2087
2422
|
});
|
|
2088
|
-
const proc = spawn([
|
|
2089
|
-
ffmpegBin,
|
|
2090
|
-
"-y",
|
|
2091
|
-
"-i",
|
|
2092
|
-
resolvedVideo,
|
|
2093
|
-
"-i",
|
|
2094
|
-
resolvedAudio,
|
|
2095
|
-
"-map",
|
|
2096
|
-
"0:v",
|
|
2097
|
-
"-map",
|
|
2098
|
-
"1:a",
|
|
2099
|
-
"-c:v",
|
|
2100
|
-
"copy",
|
|
2101
|
-
"-c:a",
|
|
2102
|
-
"aac",
|
|
2103
|
-
"-b:a",
|
|
2104
|
-
"192k",
|
|
2105
|
-
"-shortest",
|
|
2106
|
-
finalPath,
|
|
2107
|
-
], {
|
|
2423
|
+
const proc = spawn([uvPath, "tool", "install", "--upgrade", "mlx-gen"], {
|
|
2108
2424
|
stdout: "pipe",
|
|
2109
2425
|
stderr: "pipe",
|
|
2110
2426
|
});
|
|
2111
2427
|
activeProc = proc;
|
|
2112
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2113
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2428
|
+
const stdoutPromise = streamToSSE(proc.stdout, "Install", send);
|
|
2429
|
+
const stderrText = await streamToSSE(proc.stderr, "Install", send);
|
|
2114
2430
|
await stdoutPromise;
|
|
2115
2431
|
const exitCode = await proc.exited;
|
|
2116
|
-
if (exitCode === 0
|
|
2117
|
-
send("complete", {
|
|
2118
|
-
success: true,
|
|
2119
|
-
path: finalPath,
|
|
2120
|
-
filename: finalFile,
|
|
2121
|
-
});
|
|
2432
|
+
if (exitCode === 0) {
|
|
2433
|
+
send("complete", { success: true });
|
|
2122
2434
|
}
|
|
2123
2435
|
else {
|
|
2124
2436
|
send("error", {
|
|
@@ -2135,16 +2447,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2135
2447
|
res.end();
|
|
2136
2448
|
}
|
|
2137
2449
|
});
|
|
2138
|
-
// ========== MLX-Gen:
|
|
2139
|
-
app.
|
|
2140
|
-
res.json({
|
|
2141
|
-
installed: isMlxgenInstalled(),
|
|
2142
|
-
zModelDownloaded: isModelDownloaded(Z_IMAGE_MODEL),
|
|
2143
|
-
fluxModelDownloaded: isModelDownloaded(FLUX_KLEIN_MODEL),
|
|
2144
|
-
});
|
|
2145
|
-
});
|
|
2146
|
-
// ========== MLX-Gen: Install ==========
|
|
2147
|
-
app.post("/api/mlxgen/install", async (_req, res) => {
|
|
2450
|
+
// ========== MLX-Gen: Download Z-Image Model ==========
|
|
2451
|
+
app.post("/api/mlxgen/download-z-model", async (_req, res) => {
|
|
2148
2452
|
res.writeHead(200, {
|
|
2149
2453
|
"Content-Type": "text/event-stream",
|
|
2150
2454
|
"Cache-Control": "no-cache",
|
|
@@ -2154,19 +2458,20 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2154
2458
|
const send = (event, data) => {
|
|
2155
2459
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2156
2460
|
};
|
|
2461
|
+
const model = Z_IMAGE_MODEL;
|
|
2157
2462
|
try {
|
|
2158
|
-
const
|
|
2463
|
+
const mlxgen = await getMlxgenBin();
|
|
2159
2464
|
send("progress", {
|
|
2160
2465
|
status: "starting",
|
|
2161
|
-
label:
|
|
2466
|
+
label: `Downloading model ${model}...`,
|
|
2162
2467
|
});
|
|
2163
|
-
const proc = spawn([
|
|
2468
|
+
const proc = spawn([mlxgen, "download", "--model", model], {
|
|
2164
2469
|
stdout: "pipe",
|
|
2165
2470
|
stderr: "pipe",
|
|
2166
2471
|
});
|
|
2167
2472
|
activeProc = proc;
|
|
2168
|
-
const stdoutPromise = streamToSSE(proc.stdout, "
|
|
2169
|
-
const stderrText = await streamToSSE(proc.stderr, "
|
|
2473
|
+
const stdoutPromise = streamToSSE(proc.stdout, "Download", send);
|
|
2474
|
+
const stderrText = await streamToSSE(proc.stderr, "Download", send);
|
|
2170
2475
|
await stdoutPromise;
|
|
2171
2476
|
const exitCode = await proc.exited;
|
|
2172
2477
|
if (exitCode === 0) {
|
|
@@ -2187,8 +2492,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2187
2492
|
res.end();
|
|
2188
2493
|
}
|
|
2189
2494
|
});
|
|
2190
|
-
// ========== MLX-Gen: Download
|
|
2191
|
-
app.post("/api/mlxgen/download-
|
|
2495
|
+
// ========== MLX-Gen: Download FLUX.2 Klein Model ==========
|
|
2496
|
+
app.post("/api/mlxgen/download-flux-model", async (_req, res) => {
|
|
2192
2497
|
res.writeHead(200, {
|
|
2193
2498
|
"Content-Type": "text/event-stream",
|
|
2194
2499
|
"Cache-Control": "no-cache",
|
|
@@ -2198,14 +2503,13 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2198
2503
|
const send = (event, data) => {
|
|
2199
2504
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2200
2505
|
};
|
|
2201
|
-
const model = Z_IMAGE_MODEL;
|
|
2202
2506
|
try {
|
|
2203
2507
|
const mlxgen = await getMlxgenBin();
|
|
2204
2508
|
send("progress", {
|
|
2205
2509
|
status: "starting",
|
|
2206
|
-
label: `Downloading model ${
|
|
2510
|
+
label: `Downloading model ${FLUX_KLEIN_MODEL}...`,
|
|
2207
2511
|
});
|
|
2208
|
-
const proc = spawn([mlxgen, "download", "--model",
|
|
2512
|
+
const proc = spawn([mlxgen, "download", "--model", FLUX_KLEIN_MODEL], {
|
|
2209
2513
|
stdout: "pipe",
|
|
2210
2514
|
stderr: "pipe",
|
|
2211
2515
|
});
|
|
@@ -2232,8 +2536,8 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2232
2536
|
res.end();
|
|
2233
2537
|
}
|
|
2234
2538
|
});
|
|
2235
|
-
// ========== MLX-Gen: Download
|
|
2236
|
-
app.post("/api/mlxgen/download-
|
|
2539
|
+
// ========== MLX-Gen: Download SeedVR2 Model ==========
|
|
2540
|
+
app.post("/api/mlxgen/download-seedvr2-model", async (_req, res) => {
|
|
2237
2541
|
res.writeHead(200, {
|
|
2238
2542
|
"Content-Type": "text/event-stream",
|
|
2239
2543
|
"Cache-Control": "no-cache",
|
|
@@ -2247,9 +2551,9 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2247
2551
|
const mlxgen = await getMlxgenBin();
|
|
2248
2552
|
send("progress", {
|
|
2249
2553
|
status: "starting",
|
|
2250
|
-
label: `Downloading model ${
|
|
2554
|
+
label: `Downloading model ${SEEDVR2_MODEL}...`,
|
|
2251
2555
|
});
|
|
2252
|
-
const proc = spawn([mlxgen, "download", "--model",
|
|
2556
|
+
const proc = spawn([mlxgen, "download", "--model", SEEDVR2_MODEL], {
|
|
2253
2557
|
stdout: "pipe",
|
|
2254
2558
|
stderr: "pipe",
|
|
2255
2559
|
});
|
|
@@ -2285,10 +2589,63 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2285
2589
|
res.json({
|
|
2286
2590
|
installed: whichSync("hf") !== null,
|
|
2287
2591
|
ltxDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx-q8"),
|
|
2592
|
+
ltxBaseDownloaded: isModelDownloaded("dgrauet/ltx-2.3-mlx"),
|
|
2288
2593
|
ttsDownloaded: isModelDownloaded("Qwen/Qwen3-TTS-12Hz-1.7B-Base"),
|
|
2289
2594
|
mlxVlmDownloaded: isModelDownloaded(MLX_VLM_MODEL),
|
|
2290
2595
|
});
|
|
2291
2596
|
});
|
|
2597
|
+
// ========== Dots-TTS: Status + Model Download ==========
|
|
2598
|
+
app.get("/api/dots-tts/status", (_req, res) => {
|
|
2599
|
+
res.json({ downloaded: isDotsTtsModelDownloaded() });
|
|
2600
|
+
});
|
|
2601
|
+
app.post("/api/dots-tts/download-model", async (_req, res) => {
|
|
2602
|
+
res.writeHead(200, {
|
|
2603
|
+
"Content-Type": "text/event-stream",
|
|
2604
|
+
"Cache-Control": "no-cache",
|
|
2605
|
+
Connection: "keep-alive",
|
|
2606
|
+
"X-Accel-Buffering": "no",
|
|
2607
|
+
});
|
|
2608
|
+
const send = (event, data) => {
|
|
2609
|
+
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2610
|
+
};
|
|
2611
|
+
try {
|
|
2612
|
+
ensureDir(DOTS_TTS_FOLDER);
|
|
2613
|
+
send("progress", {
|
|
2614
|
+
status: "starting",
|
|
2615
|
+
label: "Downloading dots-tts model (mf-int4)...",
|
|
2616
|
+
});
|
|
2617
|
+
const proc = spawn([
|
|
2618
|
+
"hf",
|
|
2619
|
+
"download",
|
|
2620
|
+
"shraey/dots-tts-mlx",
|
|
2621
|
+
"--include",
|
|
2622
|
+
"mf-int4/*",
|
|
2623
|
+
"--local-dir",
|
|
2624
|
+
"./dots-tts-mlx-weights",
|
|
2625
|
+
], { cwd: DOTS_TTS_FOLDER, stdout: "pipe", stderr: "pipe" });
|
|
2626
|
+
activeProc = proc;
|
|
2627
|
+
const stdoutPromise = streamToSSE(proc.stdout, "dots-tts", send);
|
|
2628
|
+
const stderrText = await streamToSSE(proc.stderr, "dots-tts", send);
|
|
2629
|
+
await stdoutPromise;
|
|
2630
|
+
const exitCode = await proc.exited;
|
|
2631
|
+
if (exitCode === 0) {
|
|
2632
|
+
send("complete", { success: true });
|
|
2633
|
+
}
|
|
2634
|
+
else {
|
|
2635
|
+
send("error", {
|
|
2636
|
+
error: stderrText || `Process exited with code ${exitCode}`,
|
|
2637
|
+
exitCode,
|
|
2638
|
+
});
|
|
2639
|
+
}
|
|
2640
|
+
}
|
|
2641
|
+
catch (e) {
|
|
2642
|
+
send("error", { error: String(e) });
|
|
2643
|
+
}
|
|
2644
|
+
finally {
|
|
2645
|
+
activeProc = null;
|
|
2646
|
+
res.end();
|
|
2647
|
+
}
|
|
2648
|
+
});
|
|
2292
2649
|
app.post("/api/hf/install", async (_req, res) => {
|
|
2293
2650
|
res.writeHead(200, {
|
|
2294
2651
|
"Content-Type": "text/event-stream",
|
|
@@ -2370,6 +2727,48 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2370
2727
|
res.end();
|
|
2371
2728
|
}
|
|
2372
2729
|
});
|
|
2730
|
+
app.post("/api/hf/download-ltx-base", async (_req, res) => {
|
|
2731
|
+
res.writeHead(200, {
|
|
2732
|
+
"Content-Type": "text/event-stream",
|
|
2733
|
+
"Cache-Control": "no-cache",
|
|
2734
|
+
Connection: "keep-alive",
|
|
2735
|
+
"X-Accel-Buffering": "no",
|
|
2736
|
+
});
|
|
2737
|
+
const send = (event, data) => {
|
|
2738
|
+
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
2739
|
+
};
|
|
2740
|
+
try {
|
|
2741
|
+
send("progress", {
|
|
2742
|
+
status: "starting",
|
|
2743
|
+
label: "Downloading dgrauet/ltx-2.3-mlx...",
|
|
2744
|
+
});
|
|
2745
|
+
const proc = spawn(["hf", "download", "dgrauet/ltx-2.3-mlx"], {
|
|
2746
|
+
stdout: "pipe",
|
|
2747
|
+
stderr: "pipe",
|
|
2748
|
+
});
|
|
2749
|
+
activeProc = proc;
|
|
2750
|
+
const stdoutPromise = streamToSSE(proc.stdout, "HF Download", send);
|
|
2751
|
+
const stderrText = await streamToSSE(proc.stderr, "HF Download", send);
|
|
2752
|
+
await stdoutPromise;
|
|
2753
|
+
const exitCode = await proc.exited;
|
|
2754
|
+
if (exitCode === 0) {
|
|
2755
|
+
send("complete", { success: true });
|
|
2756
|
+
}
|
|
2757
|
+
else {
|
|
2758
|
+
send("error", {
|
|
2759
|
+
error: stderrText || `Process exited with code ${exitCode}`,
|
|
2760
|
+
exitCode,
|
|
2761
|
+
});
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
catch (e) {
|
|
2765
|
+
send("error", { error: String(e) });
|
|
2766
|
+
}
|
|
2767
|
+
finally {
|
|
2768
|
+
activeProc = null;
|
|
2769
|
+
res.end();
|
|
2770
|
+
}
|
|
2771
|
+
});
|
|
2373
2772
|
app.post("/api/hf/download-mlx-vlm", async (_req, res) => {
|
|
2374
2773
|
res.writeHead(200, {
|
|
2375
2774
|
"Content-Type": "text/event-stream",
|
|
@@ -2977,109 +3376,6 @@ export async function renderMediaRoutes({ app, getUvPath, }) {
|
|
|
2977
3376
|
writeCharacters(characters);
|
|
2978
3377
|
res.json(removed);
|
|
2979
3378
|
});
|
|
2980
|
-
// Save an extracted video frame into the project's extracted-frames folder.
|
|
2981
|
-
app.post("/api/extracted-frames", (req, res) => {
|
|
2982
|
-
const { image, filename, projectId } = req.body || {};
|
|
2983
|
-
if (!image) {
|
|
2984
|
-
res.status(400).json({ error: "Image data is required (base64)" });
|
|
2985
|
-
return;
|
|
2986
|
-
}
|
|
2987
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
2988
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
2989
|
-
return;
|
|
2990
|
-
}
|
|
2991
|
-
try {
|
|
2992
|
-
const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
|
|
2993
|
-
const buffer = Buffer.from(base64, "base64");
|
|
2994
|
-
const dir = join(EXTRACTED_FRAMES_DIR, String(projectId));
|
|
2995
|
-
ensureDir(dir);
|
|
2996
|
-
const safeName = (filename || `frame-${Date.now()}.png`).replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
2997
|
-
writeFileSync(join(dir, safeName), buffer);
|
|
2998
|
-
res.json({
|
|
2999
|
-
success: true,
|
|
3000
|
-
path: join(dir, safeName),
|
|
3001
|
-
filename: safeName,
|
|
3002
|
-
size: buffer.length,
|
|
3003
|
-
});
|
|
3004
|
-
}
|
|
3005
|
-
catch (e) {
|
|
3006
|
-
res
|
|
3007
|
-
.status(500)
|
|
3008
|
-
.json({ error: "Failed to save frame", details: String(e) });
|
|
3009
|
-
}
|
|
3010
|
-
});
|
|
3011
|
-
// Save the character sheet: current.png plus a backup copy under backup/.
|
|
3012
|
-
app.post("/api/character-sheet", (req, res) => {
|
|
3013
|
-
const { image, projectId } = req.body || {};
|
|
3014
|
-
if (!image) {
|
|
3015
|
-
res.status(400).json({ error: "Image data is required (base64)" });
|
|
3016
|
-
return;
|
|
3017
|
-
}
|
|
3018
|
-
if (!projectId || !isValidProjectId(String(projectId))) {
|
|
3019
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
3020
|
-
return;
|
|
3021
|
-
}
|
|
3022
|
-
try {
|
|
3023
|
-
const base64 = String(image).replace(/^data:[^;]+;base64,/, "");
|
|
3024
|
-
const buffer = Buffer.from(base64, "base64");
|
|
3025
|
-
const projectDir = join(CHARACTER_SHEET_DIR, String(projectId));
|
|
3026
|
-
const backupDir = join(projectDir, "backup");
|
|
3027
|
-
ensureDir(backupDir);
|
|
3028
|
-
const backupId = randomUUID();
|
|
3029
|
-
const backupPath = join(backupDir, `${backupId}.png`);
|
|
3030
|
-
const currentPath = join(projectDir, "current.png");
|
|
3031
|
-
writeFileSync(backupPath, buffer);
|
|
3032
|
-
writeFileSync(currentPath, buffer);
|
|
3033
|
-
res.json({
|
|
3034
|
-
success: true,
|
|
3035
|
-
path: currentPath,
|
|
3036
|
-
backupPath,
|
|
3037
|
-
backupFilename: `${backupId}.png`,
|
|
3038
|
-
size: buffer.length,
|
|
3039
|
-
});
|
|
3040
|
-
}
|
|
3041
|
-
catch (e) {
|
|
3042
|
-
res
|
|
3043
|
-
.status(500)
|
|
3044
|
-
.json({ error: "Failed to save character sheet", details: String(e) });
|
|
3045
|
-
}
|
|
3046
|
-
});
|
|
3047
|
-
// List saved character sheet images (current.png) for a project.
|
|
3048
|
-
app.get("/api/projects/:id/character-sheets", (req, res) => {
|
|
3049
|
-
const { id } = req.params;
|
|
3050
|
-
if (!isValidProjectId(id)) {
|
|
3051
|
-
res.status(400).json({ error: "Invalid project ID" });
|
|
3052
|
-
return;
|
|
3053
|
-
}
|
|
3054
|
-
const projectDir = join(CHARACTER_SHEET_DIR, id);
|
|
3055
|
-
const results = [];
|
|
3056
|
-
if (existsSync(projectDir)) {
|
|
3057
|
-
let entries = [];
|
|
3058
|
-
try {
|
|
3059
|
-
entries = readdirSync(projectDir);
|
|
3060
|
-
}
|
|
3061
|
-
catch {
|
|
3062
|
-
entries = [];
|
|
3063
|
-
}
|
|
3064
|
-
for (const entry of entries) {
|
|
3065
|
-
if (!entry.toLowerCase().endsWith(".png"))
|
|
3066
|
-
continue;
|
|
3067
|
-
const fullPath = join(projectDir, entry);
|
|
3068
|
-
try {
|
|
3069
|
-
if (!statSync(fullPath).isFile())
|
|
3070
|
-
continue;
|
|
3071
|
-
}
|
|
3072
|
-
catch {
|
|
3073
|
-
continue;
|
|
3074
|
-
}
|
|
3075
|
-
results.push({
|
|
3076
|
-
filename: entry,
|
|
3077
|
-
url: `/api/files?path=${encodeURIComponent(fullPath)}`,
|
|
3078
|
-
});
|
|
3079
|
-
}
|
|
3080
|
-
}
|
|
3081
|
-
res.json(results);
|
|
3082
|
-
});
|
|
3083
3379
|
// Open project folder in Finder
|
|
3084
3380
|
app.post("/api/projects/:id/open-folder", (req, res) => {
|
|
3085
3381
|
const { type } = req.body || {};
|