@effectnode/media 0.9.0 → 0.11.0

package/frontend/src/movie-app/stores/aiModelStore.ts

@@ -3,7 +3,15 @@ import { create } from "zustand";
 const API_BASE = `http://localhost:${(window as any).PORT}`;
 
 /** Models that have a dedicated download endpoint in the backend. */
-export type AiModelId = "z-image" | "flux" | "ltx" | "tts" | "gemma";
+export type AiModelId =
+  | "z-image"
+  | "flux"
+  | "seedvr2"
+  | "ltx"
+  | "ltx-base"
+  | "tts"
+  | "gemma"
+  | "dots-tts";
 
 /** Tools whose "install" step must run before their models can be downloaded. */
 export type AiToolId = "mlxgen" | "mlx-vlm" | "hf-cli";
@@ -16,9 +24,12 @@ interface AiModelStore {
   // Model download status
   zImageDownloaded: boolean | null;
   fluxDownloaded: boolean | null;
+  seedvr2Downloaded: boolean | null;
   ltxDownloaded: boolean | null;
+  ltxBaseDownloaded: boolean | null;
   ttsDownloaded: boolean | null;
   gemmaDownloaded: boolean | null;
+  dotsTtsDownloaded: boolean | null;
   // In-flight install/download (a model id or tool id)
   downloading: string | null;
   logs: string[];
@@ -68,9 +79,12 @@ async function readSSE(
 const DOWNLOAD_ENDPOINTS: Record<AiModelId, string> = {
   "z-image": "/api/mlxgen/download-z-model",
   flux: "/api/mlxgen/download-flux-model",
+  seedvr2: "/api/mlxgen/download-seedvr2-model",
   ltx: "/api/hf/download-ltx",
+  "ltx-base": "/api/hf/download-ltx-base",
   tts: "/api/hf/download-tts",
   gemma: "/api/hf/download-mlx-vlm",
+  "dots-tts": "/api/dots-tts/download-model",
 };
 
 const TOOL_ENDPOINTS: Record<AiToolId, string> = {
@@ -85,16 +99,19 @@ export const useAiModelStore = create<AiModelStore>((set, get) => ({
   hfInstalled: null,
   zImageDownloaded: null,
   fluxDownloaded: null,
+  seedvr2Downloaded: null,
   ltxDownloaded: null,
+  ltxBaseDownloaded: null,
   ttsDownloaded: null,
   gemmaDownloaded: null,
+  dotsTtsDownloaded: null,
   downloading: null,
   logs: [],
   error: null,
 
   checkStatus: async () => {
     try {
-      const [mlxgen, agent, hf] = await Promise.all([
+      const [mlxgen, agent, hf, dotsTts] = await Promise.all([
         fetch(`${API_BASE}/api/mlxgen/status`).then((r) =>
           r.ok ? r.json() : null,
         ),
@@ -102,16 +119,22 @@ export const useAiModelStore = create<AiModelStore>((set, get) => ({
           r.ok ? r.json() : null,
         ),
         fetch(`${API_BASE}/api/hf/status`).then((r) => (r.ok ? r.json() : null)),
+        fetch(`${API_BASE}/api/dots-tts/status`).then((r) =>
+          r.ok ? r.json() : null,
+        ),
       ]);
       set({
         mlxgenInstalled: mlxgen ? Boolean(mlxgen.installed) : null,
         zImageDownloaded: mlxgen ? Boolean(mlxgen.zModelDownloaded) : null,
         fluxDownloaded: mlxgen ? Boolean(mlxgen.fluxModelDownloaded) : null,
+        seedvr2Downloaded: mlxgen ? Boolean(mlxgen.seedvr2Downloaded) : null,
         mlxVlmInstalled: agent ? Boolean(agent.installed) : null,
         hfInstalled: hf ? Boolean(hf.installed) : null,
         ltxDownloaded: hf ? Boolean(hf.ltxDownloaded) : null,
+        ltxBaseDownloaded: hf ? Boolean(hf.ltxBaseDownloaded) : null,
         ttsDownloaded: hf ? Boolean(hf.ttsDownloaded) : null,
         gemmaDownloaded: hf ? Boolean(hf.mlxVlmDownloaded) : null,
+        dotsTtsDownloaded: dotsTts ? Boolean(dotsTts.downloaded) : null,
       });
     } catch {
       // Leave status unknown (null) if the checks fail.

package/frontend/src/movie-app/stores/audioToVideoStore.ts

@@ -0,0 +1,274 @@
+import { create } from "zustand";
+import type { QueueTask } from "./queueStore";
+
+const API_BASE = `http://localhost:${(window as any).PORT}`;
+
+export interface AssetFile {
+  filename: string;
+  url: string;
+}
+
+function resolveUrl(url: string): string {
+  return url.startsWith("http") ? url : `${API_BASE}${url}`;
+}
+
+interface AudioToVideoStore {
+  image: AssetFile | null;
+  audio: AssetFile | null;
+  images: AssetFile[];
+  audios: AssetFile[];
+  imagesLoading: boolean;
+  audiosLoading: boolean;
+  uploadingImage: boolean;
+  uploadingAudio: boolean;
+  steps: number;
+  duration: number;
+  frames: number;
+  prompt: string;
+  generating: boolean;
+  result: string | null;
+  error: string | null;
+
+  setImage: (a: AssetFile | null) => void;
+  setAudio: (a: AssetFile | null) => void;
+  setSteps: (n: number) => void;
+  setDuration: (d: number) => void;
+  setFrames: (n: number) => void;
+  setPrompt: (p: string) => void;
+  clearResult: () => void;
+  fetchImages: (projectId: string) => Promise<void>;
+  fetchAudios: (projectId: string) => Promise<void>;
+  uploadImage: (
+    projectId: string,
+    base64: string,
+    filename: string,
+  ) => Promise<void>;
+  uploadAudio: (
+    projectId: string,
+    base64: string,
+    filename: string,
+  ) => Promise<void>;
+  generate: (projectId: string) => Promise<void>;
+  applyQueueTask: (task: QueueTask) => void;
+  reset: () => void;
+}
+
+/** The queue task id enqueued by the current generate action, if any. */
+let audioToVideoActiveTaskId: string | null = null;
+
+async function enqueueAudioToVideoTask(
+  projectId: string,
+  imagePath: string,
+  audioPath: string,
+  prompt: string,
+  stage1Steps: number,
+  frames: number,
+): Promise<{ ok: boolean; error?: string; taskId?: string }> {
+  try {
+    const res = await fetch(`${API_BASE}/api/queue/enqueue`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        projectId,
+        type: "audio-to-video",
+        label: "Audio to video",
+        payload: { imagePath, audioPath, prompt, stage1Steps, frames },
+      }),
+    });
+    if (!res.ok) {
+      return { ok: false, error: await res.text() };
+    }
+    const task = (await res.json()) as { id?: string };
+    return { ok: true, taskId: task.id };
+  } catch (e) {
+    return { ok: false, error: String(e) };
+  }
+}
+
+export const useAudioToVideoStore = create<AudioToVideoStore>((set, get) => ({
+  image: null,
+  audio: null,
+  images: [],
+  audios: [],
+  imagesLoading: false,
+  audiosLoading: false,
+  uploadingImage: false,
+  uploadingAudio: false,
+  steps: 15,
+  duration: 1,
+  frames: 25,
+  prompt: "",
+  generating: false,
+  result: null,
+  error: null,
+
+  setImage: (image) => set({ image, error: null }),
+  setAudio: (audio) => set({ audio, error: null }),
+  setSteps: (steps) =>
+    set({ steps: Math.max(1, Math.round(Number(steps)) || 1), error: null }),
+  setFrames: (frames) =>
+    set({ frames: Math.max(1, Math.round(Number(frames)) || 1), error: null }),
+  setDuration: (duration) => {
+    const d = Math.max(0, Number(duration)) || 0;
+    set({
+      duration: d,
+      frames: Math.max(1, Math.round(d * 24) + 1),
+      error: null,
+    });
+  },
+  setPrompt: (prompt) => set({ prompt, error: null }),
+  clearResult: () => set({ result: null, error: null }),
+
+  fetchImages: async (projectId) => {
+    set({ imagesLoading: true });
+    try {
+      const res = await fetch(`${API_BASE}/api/projects/${projectId}/images`);
+      if (!res.ok) throw new Error(await res.text());
+      const images: AssetFile[] = await res.json();
+      set({
+        images: images.map((i) => ({ ...i, url: resolveUrl(i.url) })),
+        imagesLoading: false,
+      });
+    } catch {
+      set({ imagesLoading: false });
+    }
+  },
+
+  fetchAudios: async (projectId) => {
+    set({ audiosLoading: true });
+    try {
+      const res = await fetch(`${API_BASE}/api/projects/${projectId}/audios`);
+      if (!res.ok) throw new Error(await res.text());
+      const audios: AssetFile[] = await res.json();
+      set({
+        audios: audios.map((a) => ({ ...a, url: resolveUrl(a.url) })),
+        audiosLoading: false,
+      });
+    } catch {
+      set({ audiosLoading: false });
+    }
+  },
+
+  uploadImage: async (projectId, base64, filename) => {
+    set({ uploadingImage: true, error: null });
+    try {
+      const res = await fetch(`${API_BASE}/api/upload/image`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ image: base64, filename, projectId }),
+      });
+      if (!res.ok) {
+        set({ uploadingImage: false, error: await res.text() });
+        return;
+      }
+      const data = await res.json();
+      set({
+        uploadingImage: false,
+        image: {
+          filename: data.filename,
+          url: resolveUrl(
+            `/api/files?path=${encodeURIComponent(data.path)}`,
+          ),
+        },
+      });
+      void get().fetchImages(projectId);
+    } catch (e) {
+      set({ uploadingImage: false, error: String(e) });
+    }
+  },
+
+  uploadAudio: async (projectId, base64, filename) => {
+    set({ uploadingAudio: true, error: null });
+    try {
+      const res = await fetch(`${API_BASE}/api/upload/audio`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ audio: base64, filename, projectId }),
+      });
+      if (!res.ok) {
+        set({ uploadingAudio: false, error: await res.text() });
+        return;
+      }
+      const data = await res.json();
+      set({
+        uploadingAudio: false,
+        audio: {
+          filename: data.filename,
+          url: resolveUrl(
+            `/api/files?path=${encodeURIComponent(data.path)}`,
+          ),
+        },
+      });
+      void get().fetchAudios(projectId);
+    } catch (e) {
+      set({ uploadingAudio: false, error: String(e) });
+    }
+  },
+
+  generate: async (projectId) => {
+    const { image, audio, prompt, steps, frames, generating } = get();
+    if (generating || !image || !audio) return;
+
+    set({ generating: true, error: null, result: null });
+
+    const r = await enqueueAudioToVideoTask(
+      projectId,
+      image.filename,
+      audio.filename,
+      prompt.trim(),
+      steps,
+      frames,
+    );
+    audioToVideoActiveTaskId = r.taskId ?? null;
+    if (!r.ok) {
+      set({
+        generating: false,
+        error: r.error ?? "Failed to enqueue audio-to-video",
+      });
+    }
+  },
+
+  // Reconcile the audio-to-video state with the latest queue task state. Only the
+  // task enqueued by this tab is reflected, so past tasks never surface stale results.
+  applyQueueTask: (task) => {
+    if (task.type !== "audio-to-video") return;
+    if (task.id !== audioToVideoActiveTaskId) return;
+
+    if (task.status === "completed") {
+      const url = task.result?.url;
+      set({
+        generating: false,
+        result: url ? resolveUrl(url) : null,
+        error: null,
+      });
+    } else if (task.status === "failed") {
+      set({ generating: false, error: task.error ?? "Audio-to-video failed" });
+    } else if (task.status === "cancelled" || task.status === "paused") {
+      set({ generating: false, error: null });
+    } else {
+      // pending / running
+      set({ generating: true, error: null });
+    }
+  },
+
+  reset: () => {
+    audioToVideoActiveTaskId = null;
+    set({
+      image: null,
+      audio: null,
+      images: [],
+      audios: [],
+      imagesLoading: false,
+      audiosLoading: false,
+      uploadingImage: false,
+      uploadingAudio: false,
+      steps: 15,
+      duration: 1,
+      frames: 25,
+      prompt: "",
+      generating: false,
+      result: null,
+      error: null,
+    });
+  },
+}));