@effectnode/media 0.9.0 → 0.11.0

package/frontend/src/movie-app/stores/queueStore.ts

@@ -14,7 +14,12 @@ export type QueueTaskType =
   | "regenerate-asset"
   | "regenerate-video"
   | "regenerate-scene-image"
-  | "fast-image-edit";
+  | "fast-image-edit"
+  | "image-to-video"
+  | "upscale"
+  | "voice-clone"
+  | "audio-to-video"
+  | "advanced-voice-clone";
 
 export type QueueTaskStatus =
   | "pending"
@@ -74,6 +79,45 @@ function upsertTask(tasks: QueueTask[], task: QueueTask): QueueTask[] {
   return next;
 }
 
+/** Task ids already observed as completed, so we only ding once per task. */
+const completedTaskIds = new Set<string>();
+
+/** Play a short "ding" notification tone via the Web Audio API. */
+function playDing() {
+  try {
+    const Ctx =
+      window.AudioContext ||
+      (window as unknown as { webkitAudioContext: typeof AudioContext })
+        .webkitAudioContext;
+    const ctx = new Ctx();
+    const now = ctx.currentTime;
+    const osc = ctx.createOscillator();
+    const gain = ctx.createGain();
+    osc.type = "sine";
+    osc.frequency.setValueAtTime(1046.5, now); // C6
+    gain.gain.setValueAtTime(0.0001, now);
+    gain.gain.exponentialRampToValueAtTime(0.3, now + 0.01);
+    gain.gain.exponentialRampToValueAtTime(0.0001, now + 0.9);
+    osc.connect(gain);
+    gain.connect(ctx.destination);
+    osc.start(now);
+    osc.stop(now + 0.95);
+    osc.onended = () => ctx.close();
+  } catch {
+    // Ignore audio failures (e.g. blocked autoplay).
+  }
+}
+
+/** Ding once when a task first reports `completed`. */
+function maybeDing(task: QueueTask) {
+  if (task.status !== "completed") return;
+  if (completedTaskIds.has(task.id)) return;
+  completedTaskIds.add(task.id);
+  // Bound the set so a very long-lived session never grows it unboundedly.
+  if (completedTaskIds.size > 1000) completedTaskIds.clear();
+  playDing();
+}
+
 export const useQueueStore = create<QueueStore>((set, get) => ({
   tasks: [],
   loading: false,
@@ -142,6 +186,7 @@ export const useQueueStore = create<QueueStore>((set, get) => ({
       try {
         const task = JSON.parse((event as MessageEvent).data) as QueueTask;
         set((s) => ({ tasks: upsertTask(s.tasks, task) }));
+        maybeDing(task);
       } catch {
         // ignore malformed events
       }
@@ -197,6 +242,7 @@ export const useQueueStore = create<QueueStore>((set, get) => ({
         try {
           const task = JSON.parse((event as MessageEvent).data) as QueueTask;
           set((s) => ({ allTasks: upsertTask(s.allTasks, task) }));
+          maybeDing(task);
         } catch {
           // ignore malformed events
         }

package/frontend/src/movie-app/stores/upscaleStore.ts

@@ -0,0 +1,118 @@
+import { create } from "zustand";
+import type { QueueTask } from "./queueStore";
+
+const API_BASE = `http://localhost:${(window as any).PORT}`;
+
+export type UpscaleMode = "1x" | "2048";
+
+export interface UpscaleImage {
+  filename: string;
+  url: string;
+}
+
+function resolveUrl(url: string): string {
+  return url.startsWith("http") ? url : `${API_BASE}${url}`;
+}
+
+interface UpscaleStore {
+  mode: UpscaleMode;
+  image: UpscaleImage | null;
+  generating: boolean;
+  result: string | null;
+  error: string | null;
+
+  setMode: (mode: UpscaleMode) => void;
+  setImage: (image: UpscaleImage | null) => void;
+  clearResult: () => void;
+  generate: (projectId: string) => Promise<void>;
+  applyQueueTask: (task: QueueTask) => void;
+  reset: () => void;
+}
+
+/** The queue task id enqueued by the current generate action, if any. */
+let upscaleActiveTaskId: string | null = null;
+
+async function enqueueUpscaleTask(
+  projectId: string,
+  imagePath: string,
+  resolution: UpscaleMode,
+): Promise<{ ok: boolean; error?: string; taskId?: string }> {
+  try {
+    const res = await fetch(`${API_BASE}/api/queue/enqueue`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        projectId,
+        type: "upscale",
+        label: `Upscale image (${resolution})`,
+        payload: { imagePath, resolution },
+      }),
+    });
+    if (!res.ok) {
+      return { ok: false, error: await res.text() };
+    }
+    const task = (await res.json()) as { id?: string };
+    return { ok: true, taskId: task.id };
+  } catch (e) {
+    return { ok: false, error: String(e) };
+  }
+}
+
+export const useUpscaleStore = create<UpscaleStore>((set, get) => ({
+  mode: "1x",
+  image: null,
+  generating: false,
+  result: null,
+  error: null,
+
+  setMode: (mode) => set({ mode, error: null }),
+  setImage: (image) => set({ image, error: null }),
+  clearResult: () => set({ result: null, error: null }),
+
+  generate: async (projectId) => {
+    const { image, mode, generating } = get();
+    if (generating || !image) return;
+
+    set({ generating: true, error: null, result: null });
+
+    const r = await enqueueUpscaleTask(projectId, image.filename, mode);
+    upscaleActiveTaskId = r.taskId ?? null;
+    if (!r.ok) {
+      set({ generating: false, error: r.error ?? "Failed to enqueue upscale" });
+    }
+  },
+
+  // Reconcile the upscale state with the latest queue task state. Only the task
+  // enqueued by this tab is reflected, so past tasks never surface stale results.
+  applyQueueTask: (task) => {
+    if (task.type !== "upscale") return;
+    if (task.id !== upscaleActiveTaskId) return;
+
+    if (task.status === "completed") {
+      const url = task.result?.url;
+      set({
+        generating: false,
+        result: url ? resolveUrl(url) : null,
+        error: null,
+      });
+    } else if (task.status === "failed") {
+      set({ generating: false, error: task.error ?? "Upscale failed" });
+    } else if (task.status === "cancelled" || task.status === "paused") {
+      set({ generating: false, error: null });
+    } else {
+      // pending / running
+      set({ generating: true, error: null });
+    }
+  },
+
+  reset: () => {
+    upscaleActiveTaskId = null;
+    set({
+      mode: "1x",
+      image: null,
+      generating: false,
+      result: null,
+      error: null,
+    });
+  },
+}));

package/frontend/src/movie-app/stores/voiceCloneStore.ts

@@ -0,0 +1,227 @@
+import { create } from "zustand";
+import type { QueueTask } from "./queueStore";
+
+const API_BASE = `http://localhost:${(window as any).PORT}`;
+
+export type VoiceQuality = "low" | "high";
+
+export interface VoiceAudio {
+  filename: string;
+  url: string;
+}
+
+export interface GeneratedVoice {
+  id: string;
+  transcript: string;
+  quality: string;
+  refAudioFilename: string | null;
+  filename: string;
+  createdAt: string | null;
+  url: string;
+}
+
+function resolveUrl(url: string): string {
+  return url.startsWith("http") ? url : `${API_BASE}${url}`;
+}
+
+interface VoiceCloneStore {
+  quality: VoiceQuality;
+  transcript: string;
+  refAudio: VoiceAudio | null;
+  audios: VoiceAudio[];
+  audiosLoading: boolean;
+  voices: GeneratedVoice[];
+  voicesLoading: boolean;
+  uploading: boolean;
+  generating: boolean;
+  result: string | null;
+  error: string | null;
+
+  setQuality: (q: VoiceQuality) => void;
+  setTranscript: (t: string) => void;
+  setRefAudio: (a: VoiceAudio | null) => void;
+  clearResult: () => void;
+  fetchAudios: (projectId: string) => Promise<void>;
+  fetchVoices: (projectId: string) => Promise<void>;
+  uploadAudio: (
+    projectId: string,
+    base64: string,
+    filename: string,
+  ) => Promise<void>;
+  generate: (projectId: string) => Promise<void>;
+  applyQueueTask: (task: QueueTask) => void;
+  reset: () => void;
+}
+
+/** The queue task id enqueued by the current generate action, if any. */
+let voiceCloneActiveTaskId: string | null = null;
+
+/** Project whose voice-clone task should refresh the generated list on completion. */
+let voiceCloneProjectId: string | null = null;
+
+async function enqueueVoiceCloneTask(
+  projectId: string,
+  text: string,
+  refAudioPath: string,
+  quality: VoiceQuality,
+): Promise<{ ok: boolean; error?: string; taskId?: string }> {
+  try {
+    const res = await fetch(`${API_BASE}/api/queue/enqueue`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        projectId,
+        type: "voice-clone",
+        label: "Voice clone",
+        payload: { text, refAudioPath, quality },
+      }),
+    });
+    if (!res.ok) {
+      return { ok: false, error: await res.text() };
+    }
+    const task = (await res.json()) as { id?: string };
+    return { ok: true, taskId: task.id };
+  } catch (e) {
+    return { ok: false, error: String(e) };
+  }
+}
+
+export const useVoiceCloneStore = create<VoiceCloneStore>((set, get) => ({
+  quality: "high",
+  transcript: "",
+  refAudio: null,
+  audios: [],
+  audiosLoading: false,
+  voices: [],
+  voicesLoading: false,
+  uploading: false,
+  generating: false,
+  result: null,
+  error: null,
+
+  setQuality: (quality) => set({ quality, error: null }),
+  setTranscript: (transcript) => set({ transcript, error: null }),
+  setRefAudio: (refAudio) => set({ refAudio, error: null }),
+  clearResult: () => set({ result: null, error: null }),
+
+  fetchAudios: async (projectId) => {
+    set({ audiosLoading: true });
+    try {
+      const res = await fetch(`${API_BASE}/api/projects/${projectId}/audios`);
+      if (!res.ok) throw new Error(await res.text());
+      const audios: VoiceAudio[] = await res.json();
+      set({
+        audios: audios.map((a) => ({ ...a, url: resolveUrl(a.url) })),
+        audiosLoading: false,
+      });
+    } catch {
+      set({ audiosLoading: false });
+    }
+  },
+
+  fetchVoices: async (projectId) => {
+    set({ voicesLoading: true });
+    try {
+      const res = await fetch(`${API_BASE}/api/projects/${projectId}/voices`);
+      if (!res.ok) throw new Error(await res.text());
+      const voices: GeneratedVoice[] = await res.json();
+      set({
+        voices: voices.map((v) => ({ ...v, url: resolveUrl(v.url) })),
+        voicesLoading: false,
+      });
+    } catch {
+      set({ voicesLoading: false });
+    }
+  },
+
+  uploadAudio: async (projectId, base64, filename) => {
+    set({ uploading: true, error: null });
+    try {
+      const res = await fetch(`${API_BASE}/api/upload/audio`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ audio: base64, filename, projectId }),
+      });
+      if (!res.ok) {
+        set({ uploading: false, error: await res.text() });
+        return;
+      }
+      const data = await res.json();
+      set({
+        uploading: false,
+        refAudio: {
+          filename: data.filename,
+          url: resolveUrl(
+            `/api/files?path=${encodeURIComponent(data.path)}`,
+          ),
+        },
+      });
+      void get().fetchAudios(projectId);
+    } catch (e) {
+      set({ uploading: false, error: String(e) });
+    }
+  },
+
+  generate: async (projectId) => {
+    const { refAudio, transcript, quality, generating } = get();
+    if (generating || !refAudio || !transcript.trim()) return;
+
+    set({ generating: true, error: null, result: null });
+
+    voiceCloneProjectId = projectId;
+    const r = await enqueueVoiceCloneTask(
+      projectId,
+      transcript.trim(),
+      refAudio.filename,
+      quality,
+    );
+    voiceCloneActiveTaskId = r.taskId ?? null;
+    if (!r.ok) {
+      set({ generating: false, error: r.error ?? "Failed to enqueue voice clone" });
+    }
+  },
+
+  // Reconcile the voice clone state with the latest queue task state. Only the
+  // task enqueued by this tab is reflected, so past tasks never surface stale results.
+  applyQueueTask: (task) => {
+    if (task.type !== "voice-clone") return;
+    if (task.id !== voiceCloneActiveTaskId) return;
+
+    if (task.status === "completed") {
+      const url = task.result?.url;
+      set({
+        generating: false,
+        result: url ? resolveUrl(url) : null,
+        error: null,
+      });
+      if (voiceCloneProjectId) {
+        void get().fetchVoices(voiceCloneProjectId);
+      }
+    } else if (task.status === "failed") {
+      set({ generating: false, error: task.error ?? "Voice clone failed" });
+    } else if (task.status === "cancelled" || task.status === "paused") {
+      set({ generating: false, error: null });
+    } else {
+      // pending / running
+      set({ generating: true, error: null });
+    }
+  },
+
+  reset: () => {
+    voiceCloneActiveTaskId = null;
+    voiceCloneProjectId = null;
+    set({
+      quality: "high",
+      transcript: "",
+      refAudio: null,
+      audios: [],
+      audiosLoading: false,
+      voices: [],
+      voicesLoading: false,
+      uploading: false,
+      generating: false,
+      result: null,
+      error: null,
+    });
+  },
+}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectnode/media",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Start a full-stack media app: Vite + React + TypeScript frontend, Express backend with REST + WebSocket API",
   "license": "MIT",
   "author": "",