@effectnode/media 0.8.0 → 0.10.0

package/dist/backend/movie-backend/agent/prompt/script.md

@@ -0,0 +1,213 @@
+<!-- high quality -->
+
+```bash
+
+uv run mlx_audio.tts.generate \
+ --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+ --text "Ghost reporting. ... 你好嗎?" \
+ --ref_audio ./reference_voice.wav \
+ --play --output ./out --audio_format mp3 --stream --save --instruct "slow down"
+
+
+
+uv run mlx_audio.tts.generate \
+ --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+ --text "Ghost reporting. ... 你好嗎?" \
+ --ref_audio ./reference_voice.wav \
+ --play --output ./out --audio_format mp3 --stream --save --instruct "slow down"
+
+```
+
+<!-- mlx-community/Kokoro-82M-bf16 -->
+
+<!-- Low quality -->
+
+```bash
+
+uv run mlx_audio.tts.generate \
+ --model Qwen/Qwen3-TTS-12Hz-0.6B-Base \
+ --text "Ghost reporting. 中秋節快樂！ 明月幾時有？把酒問青天。不知天上宮闕，今夕是何年。我欲乘風歸去，又恐瓊樓玉宇，高處不勝寒。起舞弄清影，何似在人間！轉朱閣，低綺戶，照無眠。不應有恨，何事長向別時圓？人有悲歡離合，月有陰晴圓缺，此事古難全。但願人長久，千里共嬋娟。 中秋節快樂！" \
+ --ref_audio ./reference_voice.wav \
+ --play --output ./out --audio_format mp3 --stream --save --instruct "slow down"
+
+
+
+uv run mlx_audio.tts.generate \
+ --model Qwen/Qwen3-TTS-12Hz-0.6B-Base \
+ --text "ghost reporting! Hi how are you?" \
+ --ref_audio ./reference_voice.wav \
+ --play --output ./out --audio_format mp3 --stream --save --instruct "slow down"
+
+
+```
+
+# Image editing
+
+## 4B OK FOR COMMERCAIL USE APACHE LICENSE
+
+```bash
+
+mlxgen download --model AbstractFramework/flux.2-klein-4b-8bit
+
+mlxgen generate \
+  --image input.jpeg \
+  --prompt "The person and The ninja standing next to each other, in a studio, taking photo." \
+  --image person.png \
+  --output result.png \
+  --model AbstractFramework/flux.2-klein-4b-8bit \
+  --mlx-cache-limit-gb 20 \
+  --steps 5 --seed 42 --width 1024 --height 1024
+
+```
+
+#
+
+#
+
+# Upscale to 2048
+
+```bash
+####
+
+mlxgen download --model AbstractFramework/seedvr2-7b-8bit
+
+mlxgen upscale \
+  --model AbstractFramework/seedvr2-7b-8bit \
+  --image-path input.png \
+  --resolution 2048 \
+  --seed 42 \
+  --mlx-cache-limit-gb 100 \
+  --output input_upscaled_2048.png
+
+
+mlxgen upscale \
+  --model AbstractFramework/seedvr2-7b-8bit \
+  --image-path input.png \
+  --resolution 1x \
+  --seed 42 \
+  --mlx-cache-limit-gb 100 \
+  --output input_refined_1x.png
+
+
+```
+
+# upscale video
+
+```bash
+
+mlxgen download --model AbstractFramework/seedvr2-7b-8bit
+
+mlxgen upscale \
+  --model AbstractFramework/seedvr2-7b-8bit \
+  --video-path input.mp4 \
+  --resolution 720 \
+  --temporal-chunk-size 29 \
+  --temporal-chunk-overlap 8 \
+  --mlx-cache-limit-gb 64 \
+  --force-unsafe-video-memory \
+  --metadata \
+  --output upscalde_video.mp4
+
+
+
+mlxgen upscale \
+  --model AbstractFramework/seedvr2-7b-8bit \
+  --video-path input.mp4 \
+  --resolution 2x \
+  --mlx-cache-limit-gb 64 \
+  --force-unsafe-video-memory \
+  --output upscalde_video.mp4
+
+
+```
+
+```bash
+
+
+##
+##
+##
+
+cd to workspace/python-src/dots-tts-mlx
+
+# 1. install the quant-aware runtime (>= v0.2.0)
+pip install "git+https://github.com/sb1992/dots-tts-mlx.git@v0.2.0"
+
+# 2. download the variant you want  (use "mf-int4/*" for the faster MeanFlow decoder)
+hf download shraey/dots-tts-mlx --include "mf-int4/*" --local-dir ./dots-tts-mlx-weights
+
+# 1. install the quant-aware runtime (>= v0.2.0)
+pip install "git+https://github.com/sb1992/dots-tts-mlx.git@v0.2.0"
+
+# 2. download the variant you want  (use "mf-int4/*" for the faster MeanFlow decoder)
+hf download shraey/dots-tts-mlx --include "int4/*" --local-dir ./dots-tts-mlx-weights
+
+# 2. download the variant you want  (use "mf-int4/*" for the faster MeanFlow decoder)
+hf download shraey/dots-tts-mlx --include "mf-int4/*" --local-dir ./dots-tts-mlx-weights
+
+# 3. run (files land in ./dots-tts-mlx-weights/int4/)
+dots-tts --model ./dots-tts-mlx-weights/int4 \
+    --text "Hello from MLX." --ref-audio reference.wav --language YUE \
+    --out-path out --out-prefix clone
+
+dots-tts --model ./dots-tts-mlx-weights/mf-int4 \
+    --text "我講一個笑話比你聽. 有一日，個阿伯去茶餐廳食飯，嗌咗個「揚州炒飯」。食食吓，阿伯叫住伙計：「哥仔！你呢個揚州炒飯裡面，點解連一隻蝦都冇嘅？」伙計好冷靜咁答佢：「阿伯，咁你食『煲仔飯』嗰陣，裡面又有冇煲仔呀？」" \
+    --ref-audio reference.wav --language YUE \
+    --out-path out --out-prefix cantonese; \
+afplay ./out/cantonese_000.wav
+
+
+dots-tts --model ./dots-tts-mlx-weights/int4 \
+    --text "Ghost reporting. 我講一個笑話比你聽. 有一日，個阿伯去茶餐廳食飯，嗌咗個「揚州炒飯」。食食吓，阿伯叫住伙計：「哥仔！你呢個揚州炒飯裡面，點解連一隻蝦都冇嘅？」伙計好冷靜咁答佢：「阿伯，咁你食『煲仔飯』嗰陣，裡面又有冇煲仔呀？」" \
+    --ref-audio reference.wav --language YUE \
+    --out-path out --out-prefix fast; \
+
+afplay ./out/fast_000.wav
+
+
+dots-tts --model ./dots-tts-mlx-weights/int4 \
+    --text "Ghost reporting. 今日有d dry. 等我講一個笑話比你聽啦！ 有一日，個阿伯去茶餐廳食飯，嗌咗個「揚州炒飯」。食食吓，阿伯叫住伙計：「哥仔！你呢個揚州炒飯裡面，點解連一隻蝦都冇嘅？」伙計好冷靜咁答佢：「阿伯，咁你食『煲仔飯』嗰陣，裡面又有冇煲仔呀？」 哈哈哈哈" \
+    --ref-audio reference.wav --language YUE \
+    --out-path out --out-prefix good_int4; \
+afplay ./out/good_int4_000.wav
+
+
+dots-tts --model ./dots-tts-mlx-weights/mf-int4 \
+    --text "Ghost reporting. 今日有d dry. 等我講一個笑話比你聽啦！ 有一日，個阿伯去茶餐廳食飯，嗌咗個「揚州炒飯」。食食吓，阿伯叫住伙計：「哥仔！你呢個揚州炒飯裡面，點解連一隻蝦都冇嘅？」伙計好冷靜咁答佢：「阿伯，咁你食『煲仔飯』嗰陣，裡面又有冇煲仔呀？」 哈哈哈哈" \
+    --ref-audio reference.wav --language YUE \
+    --out-path out --out-prefix good_mf-int4; \
+
+afplay ./out/good_mf-int4_000.wav
+
+
+## not very good.
+
+# Faster 15 steps
+# cwd at the git folder
+uv run ltx-2-mlx a2v --image /Users/loklok/coder-workspace/cantonese/food.png --audio /Users/loklok/coder-workspace/cantonese/food-short.wav --frame-rate 24 --output ./out-mp4 --prompt "scene at restarurant" --stage1-steps 15 --stage2-steps 3
+
+# Standard 30 steps
+# cwd at the git folder
+
+uv run ltx-2-mlx a2v --image /Users/loklok/coder-workspace/cantonese/food.png --audio /Users/loklok/coder-workspace/cantonese/food-short.wav --frame-rate 24 --output ./out-mp4 --prompt "scene at restarurant" --stage1-steps 30 --stage2-steps 3
+
+
+uv run ltx-2-mlx a2v --image /Users/loklok/coder-workspace/cantonese/food.png --audio /Users/loklok/coder-workspace/cantonese/food-short.wav --frame-rate 24 --frames 481 --output ./out/yo.mp4 --prompt "happy food ondering story at the restarurant" --stage1-steps 8 --stage2-steps 3
+
+
+mkdir dots-tts-mlx
+git clone https://github.com/sb1992/dots-tts-mlx.git
+cd dots-tts-mlx
+uv run pip install -e .
+
+##
+##
+##
+## dgrauet/ltx-2.3-mlx (High Quality)
+## dgrauet/ltx-2.3-mlx-q8 (Standard Quality)
+##
+##
+##
+
+
+```

package/dist/backend/movie-backend/core.js

@@ -568,6 +568,39 @@ async function installPythonDependencies() {
             return false;
         }
     }
+    {
+        const pythonAppSrcDir = join(APP_DATA_DIR, "python-src");
+        if (!existsSync(pythonAppSrcDir)) {
+            mkdirSync(pythonAppSrcDir, { recursive: true });
+        }
+        const dotsTtsFolder = join(pythonAppSrcDir, "dots-tts-mlx");
+        if (!existsSync(dotsTtsFolder)) {
+            const cloneCMD = await runCommand("git", ["clone", "https://github.com/sb1992/dots-tts-mlx.git", "dots-tts-mlx"], { cwd: pythonAppSrcDir });
+            if (!cloneCMD.success) {
+                console.error("Failed to clone dots-tts-mlx:", cloneCMD.error);
+                return false;
+            }
+        }
+        // Install the dots-tts CLI (into ~/.local/bin/dots-tts, which is where
+        // `getDotsTtsBin()` resolves it). `uv tool install` does not require a venv.
+        const uvPath = await getUvPath();
+        const installResult = await runCommand(uvPath, ["tool", "install", "--editable", "."], { cwd: dotsTtsFolder });
+        if (!installResult.success) {
+            console.error("Failed to install dots-tts:", installResult.error || installResult.output);
+            return false;
+        }
+        // Install mlx-whisper (MLX-powered speech-to-text for transcription) as a
+        // `uv tool` so its `mlx_whisper` CLI lands in ~/.local/bin, mirroring dots-tts.
+        const whisperResult = await runCommand(uvPath, [
+            "tool",
+            "install",
+            "mlx-whisper",
+        ]);
+        if (!whisperResult.success) {
+            console.error("Failed to install mlx-whisper:", whisperResult.error || whisperResult.output);
+            return false;
+        }
+    }
     // {
     //   const pythonAppSrcDir = join(APP_DATA_DIR, "python-src");
     //   if (!existsSync(pythonAppSrcDir)) {

package/dist/backend/movie-backend/generation-queue.d.ts

@@ -1,5 +1,5 @@
 import { type Application } from "express";
-export type QueueTaskType = "generate" | "render" | "render-assets" | "render-videos" | "render-scene-images" | "render-asset" | "render-scene-image" | "render-video" | "regenerate-asset" | "regenerate-video" | "regenerate-scene-image";
+export type QueueTaskType = "generate" | "render" | "render-assets" | "render-videos" | "render-scene-images" | "render-asset" | "render-scene-image" | "render-video" | "regenerate-asset" | "regenerate-video" | "regenerate-scene-image" | "fast-image-edit" | "image-to-video" | "upscale" | "voice-clone" | "audio-to-video" | "advanced-voice-clone";
 export type QueueTaskStatus = "pending" | "running" | "completed" | "failed" | "cancelled" | "paused";
 export interface QueueTask {
     id: string;

package/dist/backend/movie-backend/generation-queue.js

@@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statS
 import { dirname, join } from "node:path";
 import { homedir } from "node:os";
 import { randomUUID } from "node:crypto";
-import { generateAssetImage, generateSceneImage, generateSceneVideo, cancelActiveRender, } from "./render-media.js";
+import { generateAssetImage, generateSceneImage, generateSceneVideo, generateFastImageEditImage, generateImageToVideo, generateUpscale, generateVoiceClone, generateAudioToVideo, generateAdvancedVoiceClone, cancelActiveRender, } from "./render-media.js";
 import { generateMovieStudioBible } from "./agent/agent-backend.js";
 import { movieStudioStateFile } from "./agent/workspace.js";
 const APP_DATA_DIR = join(homedir(), "media-studio");
@@ -319,7 +319,7 @@ async function runRenderSceneImages(ctx, scenes) {
             entry = { slug, ...existing };
         }
         else {
-            const r = await generateSceneImage(ctx.projectId, sc, ctx.log);
+            const r = await generateSceneImage(ctx.projectId, sc, undefined, ctx.log);
             throwIfAborted(ctx.signal);
             if ("error" in r)
                 throw new Error(r.error);
@@ -411,6 +411,82 @@ const handlers = {
         ctx.log("Production bible ready.\n");
         return result;
     },
+    // Generate a composite image via fast-image-edit (FLUX.2 Klein).
+    "fast-image-edit": async (ctx) => {
+        const { prompt, images, steps, upscale } = ctx.task.payload || {};
+        ctx.log("Generating composite image (FLUX.2 Klein)…\n");
+        const r = await generateFastImageEditImage(ctx.projectId, String(prompt || ""), Array.isArray(images) ? images : [], Number(steps), String(upscale || "none"), ctx.log);
+        if ("error" in r)
+            throw new Error(r.error);
+        return r;
+    },
+    // Generate a video from a project image via LTX-2.3 (Scene Video Generation).
+    "image-to-video": async (ctx, getUvPath) => {
+        const { prompt, imagePath, width, height, frames, frameRate, mode, stage1Steps, stage2Steps, } = ctx.task.payload || {};
+        ctx.log("Generating scene video (LTX-2.3)…\n");
+        const r = await generateImageToVideo(await getUvPath(), ctx.projectId, {
+            prompt: String(prompt || ""),
+            imagePath: String(imagePath || ""),
+            width,
+            height,
+            frames,
+            frameRate,
+            mode,
+            stage1Steps,
+            stage2Steps,
+        }, ctx.log);
+        if ("error" in r)
+            throw new Error(r.error);
+        return r;
+    },
+    // Upscale/refine a project image via mlxgen (SeedVR2).
+    "upscale": async (ctx) => {
+        const { imagePath, resolution } = ctx.task.payload || {};
+        ctx.log(`Upscaling image (${resolution || "1x"})…\n`);
+        const r = await generateUpscale(ctx.projectId, String(imagePath || ""), String(resolution || "1x"), ctx.log);
+        if ("error" in r)
+            throw new Error(r.error);
+        return r;
+    },
+    // Clone a reference voice and speak the transcript.
+    "voice-clone": async (ctx, getUvPath) => {
+        const { text, refAudioPath, quality } = ctx.task.payload || {};
+        ctx.log("Cloning voice…\n");
+        const r = await generateVoiceClone(await getUvPath(), ctx.projectId, String(text || ""), String(refAudioPath || ""), String(quality || "high"), ctx.log);
+        if ("error" in r)
+            throw new Error(r.error);
+        return r;
+    },
+    // Clone a reference voice via dots-tts.
+    "advanced-voice-clone": async (ctx) => {
+        const { text, refAudioPath, language, outPrefix, model } = ctx.task.payload || {};
+        ctx.log("Cloning voice (dots-tts)…\n");
+        const r = await generateAdvancedVoiceClone(ctx.projectId, {
+            text: String(text || ""),
+            refAudioPath: String(refAudioPath || ""),
+            language: String(language || "YUE"),
+            outPrefix: String(outPrefix || ""),
+            model: String(model || ""),
+        }, ctx.log);
+        if ("error" in r)
+            throw new Error(r.error);
+        return r;
+    },
+    // Generate a video from an image + audio via ltx-2-mlx a2v.
+    "audio-to-video": async (ctx, getUvPath) => {
+        const { imagePath, audioPath, prompt, stage1Steps, frames } = ctx.task.payload || {};
+        ctx.log("Generating audio-to-video…\n");
+        const r = await generateAudioToVideo(await getUvPath(), ctx.projectId, {
+            imagePath: String(imagePath || ""),
+            audioPath: String(audioPath || ""),
+            prompt: String(prompt || ""),
+            stage1Steps: Number(stage1Steps),
+            frames: Number(frames),
+        }, ctx.log);
+        if ("error" in r)
+            throw new Error(r.error);
+        return r;
+    },
     "render-assets": async (ctx) => {
         const { characters, places } = ctx.task.payload || {};
         return runRenderAssets(ctx, characters, places);
@@ -454,7 +530,7 @@ const handlers = {
     },
     // Generate a single scene image, skipping if it already exists.
     "render-scene-image": async (ctx) => {
-        const { scene } = ctx.task.payload || {};
+        const { scene, steps } = ctx.task.payload || {};
         if (!scene || typeof scene !== "object")
             throw new Error("scene is required");
         const s = slugify(scene?.slug);
@@ -465,7 +541,7 @@ const handlers = {
             ctx.log(`Already generated: scene-${s}.png — skipping\n`);
             return { slug: s, ...existing };
         }
-        const r = await generateSceneImage(ctx.projectId, scene, ctx.log);
+        const r = await generateSceneImage(ctx.projectId, scene, Number(steps), ctx.log);
         if ("error" in r)
             throw new Error(r.error);
         return {
@@ -533,10 +609,10 @@ const handlers = {
         };
     },
     "regenerate-scene-image": async (ctx) => {
-        const { scene } = ctx.task.payload || {};
+        const { scene, steps } = ctx.task.payload || {};
         if (!scene || typeof scene !== "object")
             throw new Error("scene is required");
-        const r = await generateSceneImage(ctx.projectId, scene, ctx.log);
+        const r = await generateSceneImage(ctx.projectId, scene, Number(steps), ctx.log);
         if ("error" in r)
             throw new Error(r.error);
         return {

package/dist/backend/movie-backend/render-media.d.ts

@@ -32,11 +32,99 @@ export declare function generateSceneVideo(uvPath: string, projectId: string, sc
 } | {
     error: string;
 }>;
+/**
+ * Generate a video from a project image via LTX-2.3. `imagePath` must be a bare
+ * filename previously uploaded/generated for this project. Used by the queue
+ * worker for the "Scene Video Generation" tab.
+ */
+export declare function generateImageToVideo(uvPath: string, projectId: string, params: {
+    prompt: string;
+    imagePath: string;
+    width?: number;
+    height?: number;
+    frames?: number;
+    frameRate?: number;
+    mode?: string;
+    model?: string;
+    stage1Steps?: number;
+    stage2Steps?: number;
+}, onLog?: (text: string) => void): Promise<{
+    filename: string;
+    url: string;
+} | {
+    error: string;
+}>;
 /**
  * Generate a single scene image via fast-image-edit (FLUX.2 Klein), using the
  * already-generated character/place images referenced by the scene's slugs.
  */
-export declare function generateSceneImage(projectId: string, scene: any, onLog?: (text: string) => void): Promise<{
+export declare function generateSceneImage(projectId: string, scene: any, steps?: number, onLog?: (text: string) => void): Promise<{
+    filename: string;
+    url: string;
+} | {
+    error: string;
+}>;
+/**
+ * Upscale/refine a project image via mlxgen (SeedVR2). `imagePath` must be a
+ * bare filename previously uploaded/generated for this project. `resolution` is
+ * either "1x" (refine at native resolution) or "2048" (upscale to 2048px).
+ */
+export declare function generateUpscale(projectId: string, imagePath: string, resolution: string, onLog?: (text: string) => void): Promise<{
+    filename: string;
+    url: string;
+} | {
+    error: string;
+}>;
+/**
+ * Clone a reference voice and speak `text` via mlx_audio.tts.generate. `refAudioPath`
+ * must be a bare filename previously uploaded to this project. `quality` is "low"
+ * or "high" (maps to a TTS model). Output is saved under <output>/voices/.
+ */
+export declare function generateVoiceClone(uvPath: string, projectId: string, text: string, refAudioPath: string, quality: string, onLog?: (text: string) => void): Promise<{
+    filename: string;
+    url: string;
+} | {
+    error: string;
+}>;
+/**
+ * Generate a video from an image + audio via ltx-2-mlx `a2v` (audio-to-video).
+ * `imagePath` and `audioPath` must be bare filenames previously uploaded/generated
+ * for this project. `stage1Steps` maps to `--stage1-steps` (15 default, 30 HD).
+ */
+export declare function generateAudioToVideo(uvPath: string, projectId: string, params: {
+    imagePath: string;
+    audioPath: string;
+    prompt: string;
+    stage1Steps: number;
+    frames: number;
+}, onLog?: (text: string) => void): Promise<{
+    filename: string;
+    url: string;
+} | {
+    error: string;
+}>;
+/**
+ * Clone a reference voice via dots-tts. Output is written to
+ * <output>/<projectId>/dots-tts/ as `<prefix>_000.wav`.
+ */
+export declare function generateAdvancedVoiceClone(projectId: string, params: {
+    text: string;
+    refAudioPath: string;
+    language: string;
+    outPrefix: string;
+    model: string;
+}, onLog?: (text: string) => void): Promise<{
+    filename: string;
+    url: string;
+} | {
+    error: string;
+}>;
+/**
+ * Generate a composite image via fast-image-edit (FLUX.2 Klein). `images` are
+ * base64 data URLs that are decoded into temp files and passed to the model as
+ * separate `--image` inputs. Used by the generation queue worker.
+ */
+export declare function generateFastImageEditImage(projectId: string, prompt: string, images: string[], steps?: number, upscaleResolution?: string, onLog?: (text: string) => void): Promise<{
     filename: string;
     url: string;
 } | {