@effectnode/media 0.9.0 → 0.11.0

package/frontend/src/movie-app/stores/generationStore.ts

@@ -16,17 +16,17 @@ export type GenerationTab =
   | "movieStudio"
   | "fastImageEdit"
   | "video"
-  | "extend"
   | "agent"
   | "storyWriter"
-  | "extract"
-  | "sceneVisual"
   | "textToImage"
   | "batchVideo"
   | "batchImageToVideo"
-  | "batchVoice"
   | "llmServer"
-  | "aiModels";
+  | "aiModels"
+  | "upscale"
+  | "voiceClone"
+  | "audioToVideo"
+  | "advancedVoiceClone";
 export type AspectRatio = "1:1" | "16:9" | "9:16" | "4:3" | "3:4";
 export type Resolution =
   | "320p"
@@ -77,16 +77,9 @@ interface VideoState {
   aspectRatio: AspectRatio;
   resolution: Resolution;
   mode: VideoMode;
-  generating: boolean;
-  result: string | null;
-  error: string | null;
-  logs: string[];
-}
-
-interface ExtendState {
-  prompt: string;
-  extendDuration: number;
-  extendFrames: number;
+  model: string;
+  stage1Steps: number;
+  stage2Steps: number;
   generating: boolean;
   result: string | null;
   error: string | null;
@@ -114,6 +107,8 @@ interface TextToImageState {
 
 interface FastImageEditState {
   prompt: string;
+  steps: number;
+  upscale: string;
   referenceImages: ProjectImage[];
   downloading: boolean;
   downloadingLogs: string[];
@@ -159,18 +154,14 @@ interface GenerationStore {
   setVideoAspectRatio: (v: AspectRatio) => void;
   setVideoResolution: (v: Resolution) => void;
   setVideoMode: (v: VideoMode) => void;
+  setVideoModel: (v: string) => void;
+  setVideoStage1Steps: (v: number) => void;
+  setVideoStage2Steps: (v: number) => void;
   clearVideoResult: () => void;
   generateVideo: (projectId: string, imagePath?: string) => Promise<void>;
+  applyVideoQueueTask: (task: QueueTask) => void;
   cancelGenerate: () => void;
 
-  // Video extension
-  extend: ExtendState;
-  setExtendPrompt: (v: string) => void;
-  setExtendDuration: (v: number) => void;
-  setExtendFrames: (v: number) => void;
-  clearExtendResult: () => void;
-  generateExtend: (projectId: string, videoPath: string) => Promise<void>;
-
   // Text-to-image (mlx-gen z-image-turbo)
   textToImage: TextToImageState;
   setTextToImagePrompt: (v: string) => void;
@@ -193,6 +184,8 @@ interface GenerationStore {
   // Fast image edit (mlx-gen FLUX.2 Klein)
   fastImageEdit: FastImageEditState;
   setFastImageEditPrompt: (v: string) => void;
+  setFastImageEditSteps: (v: number) => void;
+  setFastImageEditUpscale: (v: string) => void;
   toggleFastImageEditImage: (img: ProjectImage) => void;
   clearFastImageEditImages: () => void;
   clearFastImageEditResult: () => void;
@@ -216,8 +209,6 @@ interface GenerationStore {
   projectVideos: ProjectVideo[];
   projectVideosLoading: boolean;
   fetchProjectVideos: (projectId: string) => Promise<void>;
-  selectedVideo: ProjectVideo | null;
-  selectVideo: (video: ProjectVideo | null) => void;
 
   // Project audio picker
   projectAudios: ProjectAudio[];
@@ -393,6 +384,8 @@ async function enqueueFastImageEditTask(
   projectId: string,
   prompt: string,
   images: string[],
+  steps: number,
+  upscale: string,
 ): Promise<{ ok: boolean; error?: string; taskId?: string }> {
   try {
     const res = await fetch(`${API_BASE}/api/queue/enqueue`, {
@@ -402,7 +395,44 @@ async function enqueueFastImageEditTask(
         projectId,
         type: "fast-image-edit",
         label: "Fast image edit",
-        payload: { prompt, images, projectId },
+        payload: { prompt, images, steps, upscale, projectId },
+      }),
+    });
+    if (!res.ok) {
+      return { ok: false, error: await res.text() };
+    }
+    const task = (await res.json()) as { id?: string };
+    return { ok: true, taskId: task.id };
+  } catch (e) {
+    return { ok: false, error: String(e) };
+  }
+}
+
+/** Enqueue a scene video generation task in the backend queue worker. */
+async function enqueueImageToVideoTask(
+  projectId: string,
+  payload: {
+    prompt: string;
+    imagePath: string;
+    width: number;
+    height: number;
+    frames: number;
+    frameRate: number;
+    mode: string;
+    model: string;
+    stage1Steps: number;
+    stage2Steps: number;
+  },
+): Promise<{ ok: boolean; error?: string; taskId?: string }> {
+  try {
+    const res = await fetch(`${API_BASE}/api/queue/enqueue`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        projectId,
+        type: "image-to-video",
+        label: "Scene video",
+        payload,
       }),
     });
     if (!res.ok) {
@@ -428,6 +458,14 @@ let fastImageEditProjectId: string | null = null;
 /** The queue task id enqueued by the current generate action, if any. */
 let fastImageEditActiveTaskId: string | null = null;
 
+// ========== Scene Video Queue ==========
+
+/** Project whose image-to-video task should refresh the video grid on completion. */
+let videoProjectId: string | null = null;
+
+/** The queue task id enqueued by the current generate video action, if any. */
+let videoActiveTaskId: string | null = null;
+
 // ========== Beep ==========
 
 function playBeep() {
@@ -468,16 +506,9 @@ const initialVideo: VideoState = {
   aspectRatio: "1:1",
   resolution: "480p",
   mode: "distilled",
-  generating: false,
-  result: null,
-  error: null,
-  logs: [],
-};
-
-const initialExtend: ExtendState = {
-  prompt: "Continue the scene: the camera holds, motion flows naturally...",
-  extendDuration: 2,
-  extendFrames: 2 * 24 + 1,
+  model: "dgrauet/ltx-2.3-mlx-q8",
+  stage1Steps: 30,
+  stage2Steps: 3,
   generating: false,
   result: null,
   error: null,
@@ -530,6 +561,8 @@ const initialTextToImage: TextToImageState = {
 
 const initialFastImageEdit: FastImageEditState = {
   prompt: "",
+  steps: 4,
+  upscale: "none",
   referenceImages: [],
   downloading: false,
   downloadingLogs: [],
@@ -686,6 +719,21 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
   setVideoResolution: (resolution) =>
     set((s) => ({ video: { ...s.video, resolution } })),
   setVideoMode: (mode) => set((s) => ({ video: { ...s.video, mode } })),
+  setVideoModel: (model) => set((s) => ({ video: { ...s.video, model } })),
+  setVideoStage1Steps: (stage1Steps) =>
+    set((s) => ({
+      video: {
+        ...s.video,
+        stage1Steps: Math.max(1, Math.round(Number(stage1Steps)) || 1),
+      },
+    })),
+  setVideoStage2Steps: (stage2Steps) =>
+    set((s) => ({
+      video: {
+        ...s.video,
+        stage2Steps: Math.max(1, Math.round(Number(stage2Steps)) || 1),
+      },
+    })),
   clearVideoResult: () =>
     set((s) => ({
       video: { ...s.video, result: null, error: null, logs: [] },
@@ -735,10 +783,6 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
     // The backend now only accepts bare filenames — extract just the basename
     resolvedImagePath = resolvedImagePath.split("/").pop() || resolvedImagePath;
 
-    // Create a fresh AbortController for this single generate run
-    generateAbortController = new AbortController();
-    const signal = generateAbortController.signal;
-
     set((s) => ({
       video: {
         ...s.video,
@@ -749,75 +793,63 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
       },
     }));
 
-    try {
-      const res = await fetch(`${API_BASE}/api/render/image-to-video`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          prompt: video.prompt.trim(),
-          imagePath: resolvedImagePath,
-          projectId,
-          width,
-          height,
-          frames: video.duration * 24 + 1,
-          frameRate: 24,
-          mode: video.mode,
-        }),
-        signal,
-      });
+    videoProjectId = projectId;
+    const r = await enqueueImageToVideoTask(projectId, {
+      prompt: video.prompt.trim(),
+      imagePath: resolvedImagePath,
+      width,
+      height,
+      frames: video.duration * 24 + 1,
+      frameRate: 24,
+      mode: video.mode,
+      model: video.model,
+      stage1Steps: video.stage1Steps,
+      stage2Steps: video.stage2Steps,
+    });
+    videoActiveTaskId = r.taskId ?? null;
+    if (!r.ok) {
+      set((s) => ({
+        video: {
+          ...s.video,
+          generating: false,
+          error: r.error ?? "Failed to enqueue video",
+        },
+      }));
+    }
+  },
 
-      if (!res.ok) {
-        const err = await res.text();
-        set((s) => ({
-          video: { ...s.video, generating: false, error: err },
-        }));
-        generateAbortController = null;
-        return;
-      }
+  // Reconcile the scene video state with the latest queue task state. Only the
+  // task enqueued by this tab is reflected, so past tasks never surface stale
+  // results.
+  applyVideoQueueTask: (task) => {
+    if (task.type !== "image-to-video") return;
+    if (task.id !== videoActiveTaskId) return;
+    const v = get().video;
 
-      await readSSEStream(res, (event, data) => {
-        switch (event) {
-          case "log":
-            set((s) => ({
-              video: {
-                ...s.video,
-                logs: [...s.video.logs, data.text as string],
-              },
-            }));
-            break;
-          case "complete":
-            set((s) => ({
-              video: {
-                ...s.video,
-                generating: false,
-                result: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
-              },
-            }));
-            break;
-          case "error":
-            set((s) => ({
-              video: {
-                ...s.video,
-                generating: false,
-                error: data.error || "Video generation failed",
-              },
-            }));
-            break;
-        }
+    if (task.status === "completed") {
+      const url = task.result?.url;
+      set({
+        video: {
+          ...v,
+          generating: false,
+          result: url ? resolveImageUrl(url) : null,
+          error: null,
+        },
       });
-    } catch (e: any) {
-      // If the request was aborted, just stop silently
-      if (e?.name !== "AbortError") {
-        set((s) => ({
-          video: { ...s.video, generating: false, error: String(e) },
-        }));
-      } else {
-        set((s) => ({
-          video: { ...s.video, generating: false },
-        }));
-      }
-    } finally {
-      generateAbortController = null;
+      if (videoProjectId) get().fetchProjectVideos(videoProjectId);
+    } else if (task.status === "failed") {
+      set({
+        video: {
+          ...v,
+          generating: false,
+          error: task.error ?? "Video generation failed",
+        },
+      });
+    } else if (task.status === "cancelled" || task.status === "paused") {
+      set({ video: { ...v, generating: false, error: null } });
+    } else {
+      // pending / running
+      set({ video: { ...v, generating: true, error: null } });
     }
   },
 
@@ -1045,6 +1077,9 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
             frames: video.duration * 24 + 1,
             frameRate: 24,
             mode: video.mode,
+            model: video.model,
+            stage1Steps: video.stage1Steps,
+            stage2Steps: video.stage2Steps,
           }),
           signal,
         });
@@ -1112,140 +1147,6 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
     }
   },
 
-  // ---- Video Extension ----
-  extend: { ...initialExtend },
-
-  setExtendPrompt: (prompt) =>
-    set((s) => ({ extend: { ...s.extend, prompt, error: null } })),
-  setExtendDuration: (extendDuration) =>
-    set((s) => ({
-      extend: {
-        ...s.extend,
-        extendDuration,
-        extendFrames: extendDuration * 24 + 1,
-      },
-    })),
-  setExtendFrames: (extendFrames) =>
-    set((s) => ({
-      extend: {
-        ...s.extend,
-        extendFrames,
-        extendDuration: Math.max(0.5, (extendFrames - 1) / 24),
-      },
-    })),
-  clearExtendResult: () =>
-    set((s) => ({
-      extend: { ...s.extend, result: null, error: null, logs: [] },
-    })),
-
-  generateExtend: async (projectId, videoPath) => {
-    const { extend } = get();
-    if (!extend.prompt.trim() || extend.generating) return;
-
-    // Extract raw file path from HTTP URL (e.g. http://localhost:PORT/api/files?path=...)
-    let resolvedVideoPath = videoPath || "";
-    if (!resolvedVideoPath) {
-      set((s) => ({
-        extend: { ...s.extend, error: "Select a video to extend first." },
-      }));
-      return;
-    }
-    if (resolvedVideoPath.includes("/api/files?path=")) {
-      try {
-        const url = new URL(resolvedVideoPath);
-        resolvedVideoPath = url.searchParams.get("path") || resolvedVideoPath;
-      } catch {
-        // not a valid URL, use as-is
-      }
-    }
-    if (resolvedVideoPath.startsWith("file://")) {
-      resolvedVideoPath = resolvedVideoPath.slice(7);
-    }
-    // The backend only accepts bare filenames — extract just the basename
-    resolvedVideoPath = resolvedVideoPath.split("/").pop() || resolvedVideoPath;
-
-    // Create a fresh AbortController for this extend run
-    generateAbortController = new AbortController();
-    const signal = generateAbortController.signal;
-
-    set((s) => ({
-      extend: {
-        ...s.extend,
-        generating: true,
-        error: null,
-        result: null,
-        logs: [],
-      },
-    }));
-
-    try {
-      const res = await fetch(`${API_BASE}/api/render/extend-video`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          prompt: extend.prompt.trim(),
-          videoPath: resolvedVideoPath,
-          projectId,
-          extendFrames: extend.extendDuration * 24 + 1,
-        }),
-        signal,
-      });
-
-      if (!res.ok) {
-        const err = await res.text();
-        set((s) => ({
-          extend: { ...s.extend, generating: false, error: err },
-        }));
-        generateAbortController = null;
-        return;
-      }
-
-      await readSSEStream(res, (event, data) => {
-        switch (event) {
-          case "log":
-            set((s) => ({
-              extend: {
-                ...s.extend,
-                logs: [...s.extend.logs, data.text as string],
-              },
-            }));
-            break;
-          case "complete":
-            set((s) => ({
-              extend: {
-                ...s.extend,
-                generating: false,
-                result: `http://localhost:${(window as any).PORT}/api/files?path=${encodeURIComponent(data.path)}`,
-              },
-            }));
-            // Refresh the video grid so the new extended-*.mp4 appears
-            get().fetchProjectVideos(projectId);
-            break;
-          case "error":
-            set((s) => ({
-              extend: {
-                ...s.extend,
-                generating: false,
-                error: data.error || "Video extension failed",
-              },
-            }));
-            break;
-        }
-      });
-    } catch (e: any) {
-      // If the request was aborted, just stop silently
-      if (e?.name !== "AbortError") {
-        set((s) => ({
-          extend: { ...s.extend, generating: false, error: String(e) },
-        }));
-      } else {
-        set((s) => ({ extend: { ...s.extend, generating: false } }));
-      }
-    } finally {
-      generateAbortController = null;
-    }
-  },
-
   // ---- Fast Image Edit (FLUX.2 Klein) ----
   fastImageEdit: { ...initialFastImageEdit },
 
@@ -1254,6 +1155,17 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
       fastImageEdit: { ...s.fastImageEdit, prompt, error: null },
     })),
 
+  setFastImageEditSteps: (steps) =>
+    set((s) => ({
+      fastImageEdit: {
+        ...s.fastImageEdit,
+        steps: Math.max(1, Math.round(Number(steps)) || 1),
+      },
+    })),
+
+  setFastImageEditUpscale: (upscale) =>
+    set((s) => ({ fastImageEdit: { ...s.fastImageEdit, upscale } })),
+
   toggleFastImageEditImage: (img) =>
     set((s) => {
       const current = s.fastImageEdit.referenceImages;
@@ -1420,6 +1332,8 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
       projectId,
       fastImageEdit.prompt.trim(),
       images,
+      fastImageEdit.steps,
+      fastImageEdit.upscale,
     );
     fastImageEditActiveTaskId = r.taskId ?? null;
     if (!r.ok) {
@@ -2006,7 +1920,6 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
   // ---- Project Videos ----
   projectVideos: [],
   projectVideosLoading: false,
-  selectedVideo: null,
 
   fetchProjectVideos: async (projectId) => {
     set({ projectVideosLoading: true });
@@ -2027,17 +1940,6 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
     }
   },
 
-  selectVideo: (video) => {
-    if (!video) {
-      set({ selectedVideo: null });
-      return;
-    }
-    const fullUrl = video.url.startsWith("http")
-      ? video.url
-      : `http://localhost:${(window as any).PORT}${video.url}`;
-    set({ selectedVideo: { ...video, url: fullUrl } });
-  },
-
   // ---- Project Audio ----
   projectAudios: [],
   projectAudiosLoading: false,
@@ -2068,7 +1970,6 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
     // Immediately reset generating state so UI returns to ready
     set((s) => ({
       video: { ...s.video, generating: false },
-      extend: { ...s.extend, generating: false },
       textToImage: { ...s.textToImage, generating: false },
     }));
     // Also kill the backend spawn process
@@ -2109,11 +2010,12 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
   resetAll: () => {
     fastImageEditProjectId = null;
     fastImageEditActiveTaskId = null;
+    videoProjectId = null;
+    videoActiveTaskId = null;
     set({
       activeTab: "movieStudio",
       image: { ...initialImage },
       video: { ...initialVideo },
-      extend: { ...initialExtend },
       fastImageEdit: { ...initialFastImageEdit },
       textToImage: { ...initialTextToImage },
       textToImageProjectId: null,
@@ -2128,7 +2030,6 @@ export const useGenerationStore = create<GenerationStore>((set, get) => ({
       selectedImage: null,
       projectVideos: [],
       projectVideosLoading: false,
-      selectedVideo: null,
       projectAudios: [],
       projectAudiosLoading: false,
       csvRows: [],

package/frontend/src/movie-app/stores/movieStudioStore.ts

@@ -274,7 +274,9 @@ interface MovieStudioStore {
   sceneImagesError: string | null;
   sceneImageProgress: { current: number; total: number } | null;
   regeneratingSceneImages: string[];
+  sceneImageSteps: number;
   setIdea: (v: string) => void;
+  setSceneImageSteps: (v: number) => void;
   hydrate: (projectId: string) => Promise<void>;
   generate: (projectId: string, model: string) => Promise<void>;
   render: (projectId: string) => Promise<void>;
@@ -330,12 +332,16 @@ export const useMovieStudioStore = create<MovieStudioStore>((set, get) => ({
   sceneImagesError: null,
   sceneImageProgress: null,
   regeneratingSceneImages: [],
+  sceneImageSteps: 6,
 
   setIdea: (idea) => {
     set({ idea, error: null });
     persistMovieStudioState();
   },
 
+  setSceneImageSteps: (steps) =>
+    set({ sceneImageSteps: Math.max(1, Math.round(Number(steps)) || 1) }),
+
   hydrate: async (projectId) => {
     // Switching projects: reset to defaults so the previous project's idea
     // doesn't leak through, then load the stored state (if any) below.
@@ -435,7 +441,7 @@ export const useMovieStudioStore = create<MovieStudioStore>((set, get) => ({
         projectId,
         "render-scene-image",
         `Render scene image: ${scene.slug}`,
-        { scene, batchId },
+        { scene, batchId, steps: get().sceneImageSteps },
       );
       if (!r.ok) {
         const b = batches.get(batchId);
@@ -611,7 +617,7 @@ export const useMovieStudioStore = create<MovieStudioStore>((set, get) => ({
         projectId,
         "render-scene-image",
         `Render scene image: ${scene.slug}`,
-        { scene, batchId },
+        { scene, batchId, steps: get().sceneImageSteps },
       );
       if (!r.ok) {
         const b = batches.get(batchId);
@@ -637,7 +643,7 @@ export const useMovieStudioStore = create<MovieStudioStore>((set, get) => ({
       projectId,
       "regenerate-scene-image",
       `Regenerate scene image: ${slug}`,
-      { slug, scene },
+      { slug, scene, steps: get().sceneImageSteps },
     );
     if (!r.ok) {
       set((s) => ({
@@ -1058,6 +1064,7 @@ export const useMovieStudioStore = create<MovieStudioStore>((set, get) => ({
       sceneImagesError: null,
       sceneImageProgress: null,
       regeneratingSceneImages: [],
+      sceneImageSteps: 6,
     });
   },
 }));