@crievo/omni 0.0.0-stage → 0.1.1

package/studio/web/components/CloneVoiceDialog/useCloneVoice.ts

@@ -0,0 +1,528 @@
+/* CloneVoiceDialog's state machine: the views (list → record → save), the microphone and the
+   recorder, the attached samples and their playback, the providers and the create (or redo) call. */
+
+import { useEffect, useMemo, useRef, useState } from "react";
+import { useAvatarAudioReactive } from "../../hooks/useAvatarAudioReactive";
+import { useClonePlan } from "../../hooks/useClonePlan";
+import { api, enc, errorText, read, send } from "../../lib/api";
+import { audioDuration, createAudioAnalyser } from "../../lib/audio";
+import { shuffleAvatars } from "../../lib/avatars";
+import {
+  type CloneProviderId,
+  fileRefusal,
+  MAX_S,
+  MIN_S,
+  reorderPriority,
+  scriptWords,
+  wordStarts,
+} from "../../lib/clone-voice";
+import { blobToBase64 } from "../../lib/files";
+import { NAME_MAX, NAME_MIN } from "../../lib/voice-profiles";
+import type { CloneContext, CloneVoiceDialogProps, Result, Sample } from "./types";
+
+export type CloneVoiceState = ReturnType<typeof useCloneVoice>;
+
+export function useCloneVoice({ settings, onCreate, onClose, redo, onRedone }: CloneVoiceDialogProps) {
+  const [contexts, setContexts] = useState<CloneContext[]>([]);
+  const [listError, setListError] = useState<string | null>(null);
+  const [contextId, setContextId] = useState("mixed");
+  const [view, setView] = useState<"list" | "record" | "save">("list");
+  /* recording step 1: the language; only then the microphone */
+  const [languageOk, setLanguageOk] = useState(false);
+  const [samples, setSamples] = useState<Sample[]>([]);
+  const [microphone, setMicrophone] = useState<"no" | "ok" | "denied">("no");
+  const [level, setLevel] = useState(0);
+  /* the microphones; without permission the browser hides the names, which arrive after it */
+  const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
+  const [device, setDevice] = useState("");
+  const [recording, setRecording] = useState(false);
+  const [elapsed, setElapsed] = useState(0);
+  const [playing, setPlaying] = useState<string | null>(null);
+  const [warning, setWarning] = useState<string | null>(null);
+  const [name, setName] = useState(redo?.name ?? "");
+  const [avatars, setAvatars] = useState<string[]>(() =>
+    redo ? [redo.avatar || redo.id, ...shuffleAvatars().slice(0, 9)] : shuffleAvatars(),
+  );
+  const [avatar, setAvatar] = useState(0);
+  /* the provider open in "Configurar": its view takes the left of the "Salvar voz" step */
+  const [detail, setDetail] = useState<CloneProviderId | null>(null);
+  const [fish, setFish] = useState(true);
+  const [cartesia, setCartesia] = useState(true);
+  /* the priority among the checked providers: the first speaks, the other is fallback */
+  const [priority, setPriority] = useState<string[]>(["fish", "cartesia"]);
+  const [consent, setConsent] = useState(false);
+  /* the Cartesia plan without API cloning: the card warns and it comes unchecked */
+  const plan = useClonePlan();
+  useEffect(() => {
+    if (plan.cartesia) setCartesia(false);
+  }, [plan.cartesia]);
+  const [creating, setCreating] = useState(false);
+  const [result, setResult] = useState<Result>(null);
+
+  const mic = useRef<{ stream: MediaStream; ctx: AudioContext; timer: number } | null>(null);
+  const recorder = useRef<{ mr: MediaRecorder; start: number; timer: number } | null>(null);
+  const fileInput = useRef<HTMLInputElement>(null);
+  const player = useRef<HTMLAudioElement | null>(null);
+  const counter = useRef(0);
+  /* the center avatar moves faster while the audio plays (useAvatarAudioReactive) */
+  const avatarRoot = useRef<HTMLDivElement>(null);
+  const analyser = useRef<AnalyserNode | null>(null);
+  const totalRef = useRef(0);
+
+  /* redo: the voice's existing recording comes attached, first in the list */
+  useEffect(() => {
+    if (!redo) return;
+    let alive = true;
+    read(api.settings.clone.recording[":profile"].$get({ param: { profile: enc(redo.id) } }))
+      .then(g => {
+        if (!alive || !g.ok || !g.url) return;
+        setSamples(l =>
+          l.some(m => m.origin === "stored")
+            ? l
+            : [
+                {
+                  id: "stored",
+                  name: "Gravação anterior",
+                  origin: "stored",
+                  blob: null,
+                  url: g.url!,
+                  s: g.seconds ?? 0,
+                },
+                ...l,
+              ],
+        );
+      })
+      .catch(() => {});
+    return () => {
+      alive = false;
+    };
+  }, [redo?.id]);
+
+  useEffect(() => {
+    read(api.settings.clone.phrases.$get())
+      .then(setContexts)
+      .catch(e => setListError(errorText(e)));
+    /* already granted before: the permission button doesn't even show */
+    navigator.permissions
+      ?.query({ name: "microphone" as PermissionName })
+      .then(p => {
+        if (p.state === "granted") setMicrophone("ok");
+        if (p.state === "denied") setMicrophone("denied");
+      })
+      .catch(() => {});
+  }, []);
+
+  async function listDevices() {
+    try {
+      const all = await navigator.mediaDevices.enumerateDevices();
+      setDevices(
+        all.filter(d => d.kind === "audioinput" && d.deviceId !== "default" && d.deviceId !== "communications"),
+      );
+    } catch {
+      /* no list: the system default stays */
+    }
+  }
+  useEffect(() => {
+    listDevices();
+    navigator.mediaDevices?.addEventListener?.("devicechange", listDevices);
+    return () => navigator.mediaDevices?.removeEventListener?.("devicechange", listDevices);
+  }, []);
+
+  function closeMic() {
+    const m = mic.current;
+    if (!m) return;
+    clearInterval(m.timer);
+    m.stream.getTracks().forEach(t => t.stop());
+    m.ctx.close().catch(() => {});
+    mic.current = null;
+    setLevel(0);
+  }
+  /* leaving the screen with the mic open leaves the browser light on */
+  useEffect(
+    () => () => {
+      const g = recorder.current;
+      if (g) {
+        clearInterval(g.timer);
+        g.mr.state !== "inactive" && g.mr.stop();
+      }
+      closeMic();
+      player.current?.pause();
+    },
+    [],
+  );
+  useAvatarAudioReactive(playing === "all", avatarRoot, analyser, r =>
+    Array.from(r.querySelectorAll<HTMLCanvasElement>(".carousel-item.d0 .voice-avatar")),
+  );
+
+  /* opens the microphone and turns on the level meter; no echo cancellation nor automatic gain:
+     they change the voice the clone will learn */
+  async function openMic(id: string = device): Promise<boolean> {
+    if (mic.current) return true;
+    if (!navigator.mediaDevices?.getUserMedia) {
+      setWarning("Este navegador não deixa o app gravar. Use o Chrome, ou envie um áudio.");
+      return false;
+    }
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          ...(id ? { deviceId: { exact: id } } : {}),
+          echoCancellation: false,
+          autoGainControl: false,
+          noiseSuppression: false,
+        },
+      });
+      listDevices();
+      const ctx = new AudioContext();
+      const an = ctx.createAnalyser();
+      an.fftSize = 512;
+      ctx.createMediaStreamSource(stream).connect(an);
+      const buf = new Uint8Array(an.fftSize);
+      const timer = window.setInterval(() => {
+        an.getByteTimeDomainData(buf);
+        let sum = 0;
+        for (const v of buf) sum += ((v - 128) / 128) ** 2;
+        setLevel(Math.min(1, Math.sqrt(sum / buf.length) * 5));
+      }, 80);
+      mic.current = { stream, ctx, timer };
+      setMicrophone("ok");
+      setWarning(null);
+      return true;
+    } catch {
+      setMicrophone("denied");
+      setWarning(
+        "Não consegui abrir o microfone. Libere o acesso no navegador (cadeado na barra de endereço) e tente de novo.",
+      );
+      return false;
+    }
+  }
+
+  function openRecorder() {
+    setView("record");
+    setLanguageOk(false);
+    setWarning(null);
+  }
+
+  /* language chosen, the microphone comes: already granted before, it opens by itself */
+  async function proceed() {
+    setLanguageOk(true);
+    if (microphone === "ok") await openMic();
+  }
+
+  function close() {
+    if (recording || creating) return;
+    finish();
+  }
+
+  function finish() {
+    closeMic();
+    player.current?.pause();
+    setPlaying(null);
+    setView("list");
+    setLanguageOk(false);
+    onClose();
+  }
+
+  /* switching microphone while open reopens on the new one; closed, it only keeps the choice */
+  async function switchDevice(id: string) {
+    setDevice(id);
+    if (mic.current && !recording) {
+      closeMic();
+      await openMic(id);
+    }
+  }
+
+  /* from the form back to the list; from the microphone back to the language; from the language back to the list */
+  function back() {
+    if (recording || creating) return;
+    if (view === "save") {
+      player.current?.pause();
+      setPlaying(null);
+      setView("list");
+      return;
+    }
+    closeMic();
+    if (languageOk) {
+      setLanguageOk(false);
+      return;
+    }
+    setView("list");
+  }
+
+  function start() {
+    const m = mic.current;
+    if (!m) return;
+    const mr = new MediaRecorder(m.stream);
+    const parts: Blob[] = [];
+    mr.ondataavailable = e => e.data.size && parts.push(e.data);
+    mr.onstop = () => {
+      const g = recorder.current;
+      if (g) clearInterval(g.timer);
+      const s = g ? (performance.now() - g.start) / 1000 : 0;
+      recorder.current = null;
+      setRecording(false);
+      if (s < 1.5) {
+        setWarning("A gravação ficou curta demais: leia um trecho maior do roteiro.");
+        return;
+      }
+      const blob = new Blob(parts, { type: mr.mimeType || "audio/webm" });
+      counter.current += 1;
+      setSamples(l => [
+        ...l,
+        {
+          id: `g${Date.now()}`,
+          name: `Gravação ${counter.current}`,
+          origin: "recording",
+          blob,
+          url: URL.createObjectURL(blob),
+          s,
+        },
+      ]);
+    };
+    const startedAt = performance.now();
+    /* stops by itself at the 3 min limit, adding what was already recorded */
+    const timer = window.setInterval(() => {
+      const dt = (performance.now() - startedAt) / 1000;
+      setElapsed(dt);
+      if (totalRef.current + dt >= MAX_S && mr.state === "recording") mr.stop();
+    }, 100);
+    recorder.current = { mr, start: startedAt, timer };
+    setElapsed(0);
+    setWarning(null);
+    setRecording(true);
+    mr.start();
+  }
+
+  const stop = () => recorder.current?.mr.stop();
+
+  function openSave() {
+    closeMic();
+    setWarning(null);
+    setResult(null);
+    setView("save");
+  }
+
+  async function onPickFiles(list: FileList | null) {
+    const added: Sample[] = [];
+    const refused: string[] = [];
+    for (const f of Array.from(list ?? [])) {
+      const why = fileRefusal(f);
+      if (why) {
+        refused.push(why);
+        continue;
+      }
+      const url = URL.createObjectURL(f);
+      const s = await audioDuration(url);
+      const already = totalRef.current + added.reduce((x, n) => x + n.s, 0);
+      if (already + s > MAX_S) {
+        URL.revokeObjectURL(url);
+        refused.push(`${f.name} passaria do limite de 3 min`);
+        continue;
+      }
+      added.push({ id: `a${Date.now()}${added.length}`, name: f.name, origin: "file", blob: f, url, s });
+    }
+    if (added.length) {
+      setSamples(l => [...l, ...added]);
+      setView("list");
+      closeMic();
+    }
+    setWarning(refused.length ? `${refused.join(" · ")}. Máximo de 10 MB por arquivo e de 3 min no total.` : null);
+  }
+
+  function remove(id: string) {
+    setSamples(l => {
+      const x = l.find(m => m.id === id);
+      if (x?.blob) URL.revokeObjectURL(x.url);
+      return l.filter(m => m.id !== id);
+    });
+    if (playing === id) {
+      player.current?.pause();
+      setPlaying(null);
+    }
+  }
+
+  function listen(m: { id: string; url: string }) {
+    if (playing === m.id) {
+      player.current?.pause();
+      setPlaying(null);
+      return;
+    }
+    player.current?.pause();
+    const p = new Audio(m.url);
+    p.onended = () => setPlaying(null);
+    player.current = p;
+    setPlaying(m.id);
+    p.play().catch(() => setPlaying(null));
+  }
+
+  /* the whole attached audio, samples in order */
+  function listenAll() {
+    player.current?.pause();
+    if (playing === "all") {
+      setPlaying(null);
+      return;
+    }
+    const queue = [...samples];
+    const play = (i: number) => {
+      if (i >= queue.length) {
+        setPlaying(null);
+        return;
+      }
+      const p = new Audio(queue[i].url);
+      p.onended = () => play(i + 1);
+      analyser.current = createAudioAnalyser(p);
+      player.current = p;
+      p.play().catch(() => setPlaying(null));
+    };
+    setPlaying("all");
+    play(0);
+  }
+
+  const ctx = contexts.find(c => c.id === contextId) ?? contexts[0];
+  const words = useMemo(() => scriptWords(ctx?.phrases ?? []), [ctx]);
+  const starts = useMemo(() => wordStarts(words), [words]);
+  /* how many words were already called; outside recording, none */
+  const called = recording ? starts.filter(t => t <= elapsed).length : 0;
+  const hasFish = !!(settings.voice.fish.key.masked || settings.voice.fish.key.in_env);
+  const hasCartesia = !!(settings.voice.cartesia.key.masked || settings.voice.cartesia.key.in_env);
+  const total = samples.reduce((s, m) => s + m.s, 0);
+  totalRef.current = total;
+  const measured = total + (recording ? elapsed : 0);
+  const filesWithoutDuration = samples.some(m => m.origin === "file" && m.s === 0);
+  const enough = total >= MIN_S || (filesWithoutDuration && samples.length > 0);
+  const providers = [fish && hasFish && "fish", cartesia && hasCartesia && "cartesia"].filter(Boolean) as string[];
+  const order = priority.filter(id => providers.includes(id));
+  const moveProvider = (i: number, d: -1 | 1) => setPriority(reorderPriority(order, priority, i, d));
+  /* an old voice may have a name longer than the new limit: it still counts, it just doesn't grow */
+  const maxName = Math.max(NAME_MAX, redo?.name.length ?? 0);
+  const nameValid = name.trim().length >= NAME_MIN && name.trim().length <= maxName;
+  /* redo: the providers the voice already has; consent only applies to new audio */
+  const redoOn = redo ? (["fish", "cartesia", "elevenlabs"] as const).filter(p => redo[p]) : [];
+  const hasNew = samples.some(m => m.blob);
+  const canSave = redo
+    ? !creating && nameValid && enough && redoOn.length > 0 && (!hasNew || consent)
+    : !creating && nameValid && consent && providers.length > 0;
+
+  async function create() {
+    if (!ctx) return;
+    setCreating(true);
+    setResult(null);
+    try {
+      /* a continuous recording goes without `text`: what was read may stop mid-script, and Fish
+         transcribes by itself. The server only sends `texts` when all have one */
+      const fresh = samples.filter((m): m is Sample & { blob: Blob } => !!m.blob);
+      const sent = await Promise.all(fresh.map(async m => ({ mime: m.blob.type, data: await blobToBase64(m.blob) })));
+      const r = (
+        redo
+          ? await send(
+              api.settings.clone.redo.$post({
+                json: {
+                  profile: redo.id,
+                  name: name.trim(),
+                  avatar: avatars[avatar],
+                  context: ctx.id,
+                  samples: sent,
+                  /* the previous recording stayed in the list: joins; deleted: records everything again */
+                  keep: samples.some(m => m.origin === "stored"),
+                  consent,
+                },
+              }),
+            )
+          : await send(
+              api.settings.clone.$post({
+                json: {
+                  name: name.trim(),
+                  avatar: avatars[avatar],
+                  context: ctx.id,
+                  providers,
+                  order,
+                  samples: sent,
+                  consent,
+                },
+              }),
+            )
+      ) as NonNullable<Result>;
+      setResult(r);
+      if (r.ok) {
+        samples.forEach(m => m.blob && URL.revokeObjectURL(m.url));
+        setSamples([]);
+        setName("");
+        setConsent(false);
+        setAvatars(shuffleAvatars());
+        setAvatar(0);
+        finish();
+        if (redo) onRedone?.([r.output ?? "", ...(r.warnings ?? [])].filter(Boolean).join(" · "));
+        else onCreate?.(r.failed?.length ? [r.output ?? "", ...(r.warnings ?? [])].filter(Boolean) : []);
+      }
+    } catch (e) {
+      setResult({ ok: false, error: errorText(e) });
+    } finally {
+      setCreating(false);
+    }
+  }
+
+  return {
+    contexts,
+    listError,
+    setContextId,
+    view,
+    languageOk,
+    samples,
+    microphone,
+    level,
+    devices,
+    device,
+    recording,
+    elapsed,
+    playing,
+    warning,
+    name,
+    setName,
+    avatars,
+    setAvatars,
+    avatar,
+    setAvatar,
+    detail,
+    setDetail,
+    fish,
+    setFish,
+    cartesia,
+    setCartesia,
+    consent,
+    setConsent,
+    plan,
+    creating,
+    result,
+    fileInput,
+    avatarRoot,
+    openMic,
+    openRecorder,
+    proceed,
+    close,
+    switchDevice,
+    back,
+    start,
+    stop,
+    openSave,
+    onPickFiles,
+    remove,
+    listen,
+    listenAll,
+    ctx,
+    words,
+    called,
+    hasFish,
+    hasCartesia,
+    total,
+    measured,
+    filesWithoutDuration,
+    enough,
+    order,
+    moveProvider,
+    maxName,
+    nameValid,
+    redoOn,
+    hasNew,
+    canSave,
+    create,
+  };
+}

package/studio/web/components/ConnectionSignals.tsx

@@ -0,0 +1,51 @@
+/* the services, queue and live state in one line (top bar) */
+
+import { Link } from "@tanstack/react-router";
+import { useConnected } from "../hooks/useConnected";
+import { useQueue } from "../hooks/useQueue";
+import { useServices } from "../hooks/useServices";
+import { TOOL_LABEL } from "../lib/format";
+import { premiereStatus, swapStatus } from "../lib/services";
+
+export function ConnectionSignals() {
+  const s = useServices();
+  const queue = useQueue();
+  const connected = useConnected();
+  const p = s ? premiereStatus(s.premiere, !!queue?.running.some(t => t.tool === "premiere-export")) : null;
+  const m = s ? swapStatus(s.memory) : null;
+  return (
+    <div className="signals">
+      {s && p && m && (
+        <>
+          <span className="signal" title={s.voicestudio.url}>
+            <span className={`point ${s.voicestudio.ok ? "ok" : "error"}`} />
+            VoiceStudio
+          </span>
+          <span className="signal" title={p.text}>
+            <span className={`point ${p.variant}`} />
+            Premiere
+          </span>
+          <span className="signal" title={`Pressão ${s.memory.pressure}`}>
+            <span className={`point ${m.variant}`} />
+            Swap {Math.round(m.fraction * 100)}%
+          </span>
+        </>
+      )}
+      {queue && (
+        <Link to="/queue" className="signal">
+          <span className={`point ${queue.running.length ? "running" : "neutral"}`} />
+          {queue.running.length
+            ? `${TOOL_LABEL[queue.running[0].tool] ?? queue.running[0].tool} rodando`
+            : "Fila livre"}
+          {queue.waiting.length ? ` · ${queue.waiting.length} esperando` : ""}
+        </Link>
+      )}
+      <span
+        className={`signal live ${connected ? "ok" : "error"}`}
+        title={connected ? "Atualizando ao vivo" : "Sem conexão com o studio"}
+      >
+        {connected ? "Ao vivo" : "desconectado"}
+      </span>
+    </div>
+  );
+}

package/studio/web/components/EditVoiceDialog/EditVoiceFooter.tsx

@@ -0,0 +1,72 @@
+import { ArrowClockwise, Check, Trash, X } from "../icons";
+
+/* the edit frame's bar: remove (with the confirmation), "Refazer voz" and save */
+export function EditVoiceFooter({
+  hasProfile,
+  isNew,
+  confirmRemove,
+  onConfirmRemoveChange,
+  canRedo,
+  saving,
+  canSave,
+  onRemove,
+  onRedo,
+  onSave,
+}: {
+  /* an existing voice (only it can be removed or redone) */
+  hasProfile: boolean;
+  isNew: boolean;
+  confirmRemove: boolean;
+  onConfirmRemoveChange: (v: boolean) => void;
+  /* the voice has a clone on some provider */
+  canRedo: boolean;
+  saving: boolean;
+  canSave: boolean;
+  onRemove: () => void;
+  onRedo: () => void;
+  onSave: () => void;
+}) {
+  return (
+    <footer className="clone-bar save">
+      {hasProfile &&
+        (confirmRemove ? (
+          <>
+            <button type="button" className="button danger remove" disabled={saving} onClick={onRemove}>
+              <Trash className="ic" /> {saving ? "Removendo…" : "Remover mesmo"}
+            </button>
+            <button
+              type="button"
+              className="button remove-cancel"
+              disabled={saving}
+              onClick={() => onConfirmRemoveChange(false)}
+            >
+              <X className="ic" /> Cancelar
+            </button>
+          </>
+        ) : (
+          <button
+            type="button"
+            className="button destructive remove"
+            disabled={saving}
+            onClick={() => onConfirmRemoveChange(true)}
+          >
+            <Trash className="ic" /> Remover
+          </button>
+        ))}
+      {hasProfile && canRedo && !confirmRemove && (
+        <button type="button" className="button" disabled={saving} onClick={onRedo}>
+          <ArrowClockwise className="ic" /> Refazer voz
+        </button>
+      )}
+      <button type="button" className="button primary" disabled={!canSave} onClick={onSave}>
+        {saving ? (
+          "Salvando…"
+        ) : (
+          <>
+            <Check weight="bold" className="ic" /> {isNew ? "Adicionar voz" : "Salvar voz"}
+          </>
+        )}
+      </button>
+    </footer>
+  );
+}