@crievo/omni 0.0.0-stage → 0.1.0

package/studio/web/hooks/useMissingDms.ts

@@ -0,0 +1,211 @@
+/* The "Sem DM" panel's state (MissingDmsPanel): the list read from the post, the checked
+   comments, the send mode and text, the batch confirmation and the focused comment. */
+
+import { useState } from "react";
+import { send as request, type ActionResult } from "../lib/api";
+import { ruleApi, type RuleTargetRef } from "../lib/omnichat-api";
+import {
+  askingItems,
+  missingBeforeSend,
+  replyTarget,
+  replyText,
+  type SendMode,
+  sendActionLabel,
+} from "../lib/missing-dms";
+import type { ManualSend, MissingDmList } from "../types/api";
+import { PAGE_SIZE } from "../components/RuleLog";
+
+type SendResult = NonNullable<ActionResult> & { results?: ManualSend[] };
+
+export function useMissingDms({
+  target,
+  noRule,
+  hasPublicReplies,
+  onSend,
+}: {
+  target: RuleTargetRef;
+  noRule: boolean;
+  hasPublicReplies: boolean;
+  onSend: () => void;
+}) {
+  const [list, setList] = useState<MissingDmList | null>(null);
+  const [checked, setChecked] = useState<Set<string>>(new Set());
+  const [mode, setMode] = useState<SendMode>(hasPublicReplies ? "both" : "dm");
+  const [text, setText] = useState("");
+  const [confirm, setConfirm] = useState(false);
+  const [busy, setBusy] = useState<"fetch" | "send" | "delete" | null>(null);
+  const [result, setResult] = useState<SendResult | null>(null);
+  const [showAll, setShowAll] = useState(false);
+  /* the focused comment (Responder on the row), its text and whether the rule's DM goes too */
+  const [focus, setFocus] = useState<string | null>(null);
+  const [focusText, setFocusText] = useState("");
+  const [focusDm, setFocusDm] = useState(false);
+  /* the feed: every comment or only the ones without the DM; the comment waiting for the second click on Apagar */
+  const [filter, setFilter] = useState<"all" | "nodm">("nodm");
+  const [askDelete, setAskDelete] = useState<string | null>(null);
+
+  const commentOnly = mode === "comment";
+
+  /* the list read, or why not (Omni reads what it returns: lib/omni-ui omnichat_comments) */
+  const fetchList = async (clear = true): Promise<{ list: MissingDmList } | { error: string }> => {
+    setBusy("fetch");
+    if (clear) setResult(null);
+    setConfirm(false);
+    const r = await request(ruleApi(target).comments());
+    setBusy(null);
+    if (!r.ok || !r.list) {
+      setResult(r);
+      return { error: (!r.ok && r.error) || "não deu para ler os comentários" };
+    }
+    const l = r.list;
+    setList(l);
+    setChecked(m => new Set([...m].filter(id => l.items.some(i => i.id === id && (commentOnly || i.within_deadline)))));
+    /* the comment being answered or deleted left the post */
+    setFocus(f => (f && replyTarget(l.thread, f) ? f : null));
+    setAskDelete(d => (d && replyTarget(l.thread, d) ? d : null));
+    return { list: l };
+  };
+  const openFocus = (id: string) => {
+    setFocus(id);
+    setAskDelete(null);
+    setFocusText("");
+    setFocusDm(false);
+    setConfirm(false);
+    setResult(null);
+  };
+  const replyFocus = async () => {
+    const to = replyTarget(list?.thread, focus);
+    if (!to) return;
+    setBusy("send");
+    setResult(null);
+    /* the DM is the comment's: a reply to a reply is only ever a comment */
+    const r = await request(
+      ruleApi(target).send({
+        ids: [to.parentId],
+        public_reply: true,
+        public_reply_only: to.nested || !focusDm,
+        text: replyText(to, focusText.trim()),
+      }),
+    );
+    setBusy(null);
+    setResult(r);
+    /* what did not go stays focused, with its text, to try again */
+    if (r.ok && !r.results?.some(x => x.state !== "done")) {
+      setFocus(null);
+      setFocusText("");
+    }
+    if (r.ok) {
+      onSend();
+      await fetchList(false);
+    }
+  };
+  /* Apagar: the first click only asks (askDelete); the second deletes on Instagram, with no way back */
+  const remove = async (id: string) => {
+    setBusy("delete");
+    setResult(null);
+    const r = await request(ruleApi(target).remove([id]));
+    setBusy(null);
+    setAskDelete(null);
+    setResult(r);
+    if (r.ok) {
+      setChecked(m => new Set([...m].filter(x => x !== id)));
+      setFocus(f => (f === id ? null : f));
+      onSend();
+      await fetchList(false);
+    }
+  };
+  const send = async () => {
+    setBusy("send");
+    setResult(null);
+    const r = await request(
+      ruleApi(target).send({
+        ids: [...checked],
+        public_reply: mode !== "dm",
+        public_reply_only: commentOnly,
+        text: mode === "dm" ? "" : text.trim(),
+      }),
+    );
+    setBusy(null);
+    setConfirm(false);
+    setResult(r);
+    if (r.ok) {
+      setChecked(new Set());
+      setText("");
+      onSend();
+      await fetchList(false);
+    }
+  };
+  const check = (id: string, yes: boolean) => {
+    setChecked(m => {
+      const n = new Set(m);
+      yes ? n.add(id) : n.delete(id);
+      return n;
+    });
+    setConfirm(false);
+  };
+  /* without DM there is no deadline; back to the DM, who passed 7 days leaves the selection */
+  const changeMode = (m: SendMode) => {
+    setMode(m);
+    setConfirm(false);
+    if (m !== "comment")
+      setChecked(s => new Set([...s].filter(id => list?.items.some(i => i.id === id && i.within_deadline))));
+  };
+  const written = text.trim();
+  const asking = askingItems(list?.items, commentOnly);
+  const replying = replyTarget(list?.thread, focus);
+  /* what the feed lists: the whole post, or the comments without the DM (the ones the batch can pick) */
+  const without = new Set(list?.items.map(i => i.id));
+  const feedAll = (list?.thread ?? []).filter(c => filter === "all" || without.has(c.id));
+  const feed = showAll ? feedAll : feedAll.slice(0, PAGE_SIZE);
+  const focusWritten = focusText.trim();
+  const n = checked.size;
+  const missing = missingBeforeSend({ n, commentOnly, written, noRule });
+  const action = sendActionLabel(commentOnly, n);
+  const problems = result?.results?.filter(x => x.state !== "done") ?? [];
+
+  return {
+    list,
+    checked,
+    setChecked,
+    mode,
+    text,
+    setText,
+    confirm,
+    setConfirm,
+    busy,
+    result,
+    showAll,
+    setShowAll,
+    focus,
+    setFocus,
+    focusText,
+    setFocusText,
+    focusDm,
+    setFocusDm,
+    commentOnly,
+    fetchList,
+    openFocus,
+    replyFocus,
+    send,
+    check,
+    changeMode,
+    written,
+    asking,
+    replying,
+    filter,
+    setFilter,
+    askDelete,
+    setAskDelete,
+    remove,
+    without,
+    feedAll,
+    feed,
+    focusWritten,
+    n,
+    missing,
+    action,
+    problems,
+  };
+}
+
+export type MissingDms = ReturnType<typeof useMissingDms>;

package/studio/web/hooks/useMissingSetupSteps.ts

@@ -0,0 +1,26 @@
+/* The required getting-started steps still missing (null while checking); a new `key` checks
+   again. The menu hides "Primeiros passos" when none is missing, and the overview says what is. */
+
+import { useEffect, useRef } from "react";
+import { api } from "../lib/api";
+import { invalidate, queryKeys } from "../lib/query-client";
+import type { Onboarding } from "../types/api";
+import { useLiveQuery } from "./useLiveQuery";
+
+export function useMissingSetupSteps(key?: string): string[] | null {
+  const { data: onboarding, error } = useLiveQuery(queryKeys.onboarding, () => api.onboarding.$get({ query: {} }));
+  const data: Onboarding | null = onboarding;
+  const first = useRef(true);
+  useEffect(() => {
+    if (first.current) {
+      first.current = false;
+      return;
+    }
+    void invalidate(queryKeys.onboarding);
+  }, [key]);
+  if (error) return [];
+  if (!data) return null;
+  return data.checked
+    ? data.steps.filter(s => !s.optional && s.state === "missing").map(s => s.title.replace(/^\d+\.\s*/, ""))
+    : [];
+}

package/studio/web/hooks/useNow.ts

@@ -0,0 +1,11 @@
+import { useEffect, useState } from "react";
+
+/* a clock for elapsed times that tick on screen */
+export function useNow(ms = 1000) {
+  const [now, setNow] = useState(Date.now());
+  useEffect(() => {
+    const t = setInterval(() => setNow(Date.now()), ms);
+    return () => clearInterval(t);
+  }, [ms]);
+  return now;
+}

package/studio/web/hooks/useOmniAgent.ts

@@ -0,0 +1,278 @@
+/* Omni's page state: the config, voices and measurements (GET /api/agent) and the live conversation:
+   Grok Voice (lib/omni-realtime) or the cascade, STT → LLM → Fish (lib/omni-cascade), picked per
+   conversation. One conversation at a time; leaving the page hangs up.
+
+   The conversation is saved as it goes (api/services/agent-history.ts keeps it only once it had a
+   genuine request) under its own id, which a continued conversation keeps: the Omni panel's
+   Conversas reopens it and `start(mode, earlier)` goes on from where it stopped. */
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import { api, enc, send } from "../lib/api";
+import { invalidate, queryKeys } from "../lib/query-client";
+import { OmniConversation, type OmniEntry, type OmniStatus } from "../lib/omni-realtime";
+import { type CascadeMode, OmniCascade } from "../lib/omni-cascade";
+import { openScreen, refreshMissingDms } from "../lib/omni-ui";
+import type { AgentInfo, AgentPendingAction, AgentResolvedAction } from "../types/api";
+import type { NavigateArgs } from "../../shared/omni-screens";
+import type { Conversation, ConversationAction } from "../../shared/schemas/agent-history";
+import { useLiveQuery } from "./useLiveQuery";
+
+export function useAgentInfo() {
+  const { data, error, reload } = useLiveQuery(queryKeys.agent, () => api.agent.$get());
+  const info: AgentInfo | null = data;
+  return { info, error, reload };
+}
+
+/* grok: Grok Voice, billed per open minute; stream / chunks: the cascade, billed per speech */
+export type OmniMode = "grok" | CascadeMode;
+
+/* a saved conversation, as the Conversas list opens it */
+export type OmniConversationRecord = Conversation;
+
+const newConversationId = () =>
+  `${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "")}-${crypto.randomUUID().slice(0, 6)}`;
+
+/* opens the screen of an action (best effort: the card and the voice already said what happens) */
+const show = (open: NavigateArgs | null | undefined) => {
+  if (open) void openScreen(open);
+};
+
+/* what the screen does once a card went: the "Sem DM" list drops the ones that got the send */
+const afterConfirm = (a: { kind: string; slug?: string | null; payload?: Record<string, unknown> }) => {
+  if (a.kind === "omnichat_send" && a.slug)
+    void refreshMissingDms(a.slug, a.payload?.content === "carousel" ? "carousel" : "video");
+};
+
+/* while a call is live, its record is saved this long after the last change */
+const SAVE_DEBOUNCE_MS = 3000;
+
+export function useOmniConversation(opts: { onQuota?: () => void } = {}) {
+  const onQuota = useRef(opts.onQuota);
+  onQuota.current = opts.onQuota;
+  const [status, setStatus] = useState<OmniStatus>("idle");
+  const [entries, setEntries] = useState<OmniEntry[]>([]);
+  const [pending, setPending] = useState<AgentPendingAction[]>([]);
+  /* read when the call ends, from a callback made before the cards arrived */
+  const pendingNow = useRef<AgentPendingAction[]>([]);
+  pendingNow.current = pending;
+  const [error, setError] = useState<string | null>(null);
+  const [muted, setMutedState] = useState(false);
+  const [startedAt, setStartedAt] = useState<number | null>(null);
+  const [mode, setMode] = useState<OmniMode>("grok");
+  const conv = useRef<OmniConversation | OmniCascade | null>(null);
+  /* the analyser of the reply audio, for the avatar (useAvatarAudioReactive reads .current) */
+  const analyser = useRef<AnalyserNode | null>(null);
+  /* the user's microphone, for the avatar while Omni listens */
+  const micAnalyser = useRef<AnalyserNode | null>(null);
+  /* the record being saved: its id, when it began, the voice sessions it used and each card's fate */
+  const record = useRef<{
+    id: string;
+    mode: OmniMode;
+    started_at: string;
+    sessions: Set<string>;
+    actions: Map<string, ConversationAction>;
+  } | null>(null);
+  /* the transcript as the class last reported it (the state lags a render behind) */
+  const entriesNow = useRef<OmniEntry[]>([]);
+  const saveTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  const save = useCallback(() => {
+    if (saveTimer.current) clearTimeout(saveTimer.current);
+    saveTimer.current = null;
+    const r = record.current;
+    if (!r) return;
+    const sid = conv.current?.sessionId;
+    if (sid) r.sessions.add(sid);
+    void send(
+      api.agent.conversations[":id"].$post({
+        param: { id: enc(r.id) },
+        json: {
+          mode: r.mode,
+          sessions: [...r.sessions],
+          entries: entriesNow.current,
+          actions: [...r.actions.values()],
+          started_at: r.started_at,
+        },
+      }),
+    );
+  }, []);
+  const saveSoon = useCallback(() => {
+    if (saveTimer.current) clearTimeout(saveTimer.current);
+    saveTimer.current = setTimeout(save, SAVE_DEBOUNCE_MS);
+  }, [save]);
+  const track = (
+    a: { id: string; kind: string; slug?: string | null; summary: string },
+    state: ConversationAction["state"],
+  ) => {
+    record.current?.actions.set(a.id, {
+      id: a.id,
+      kind: a.kind,
+      slug: a.slug ?? null,
+      summary: a.summary,
+      state,
+      at: new Date().toISOString(),
+    });
+  };
+
+  /* `earlier`: a saved conversation to continue (its transcript and cards come back, same record).
+     `opts`: a call the panel opened by itself (OmniWidget) says the recorded greeting through the call
+     (greet: the voice's gender), and goes without the engine's own silence handling until the user
+     speaks (holdIdle) */
+  const start = useCallback(
+    async (
+      mode: OmniMode = "grok",
+      earlier?: OmniConversationRecord,
+      opts: { greet?: string; holdIdle?: boolean } = {},
+    ) => {
+      conv.current?.stop();
+      setError(null);
+      const history: OmniEntry[] = earlier
+        ? [
+            ...(earlier.entries as OmniEntry[]),
+            {
+              kind: "note",
+              id: `continue-${Date.now()}`,
+              text: `Continuando a conversa de ${new Date(earlier.updated_at).toLocaleString("pt-BR", { dateStyle: "short", timeStyle: "short" })}.`,
+            },
+          ]
+        : [];
+      record.current = {
+        id: earlier?.id ?? newConversationId(),
+        mode,
+        started_at: earlier?.started_at ?? new Date().toISOString(),
+        sessions: new Set(earlier?.sessions ?? []),
+        actions: new Map((earlier?.actions ?? []).map(a => [a.id, a])),
+      };
+      entriesNow.current = history;
+      setEntries(history);
+      setPending([]);
+      setMutedState(false);
+      const callbacks = {
+        onStatus: (s: OmniStatus) => {
+          setStatus(s);
+          if (s === "idle" || s === "error") {
+            setStartedAt(null);
+            /* hanging up cancels what still waits for Confirmar, and the cards leave the screen (the
+             server does the same when the call's end arrives, also if this tab closes) */
+            for (const a of pendingNow.current) {
+              void send(api.agent.actions[":id"].cancel.$post({ param: { id: enc(a.id) }, json: {} }));
+              track(a, "canceled");
+            }
+            setPending([]);
+            save();
+            /* the measurements include the conversation that just ended (also the idle hang-up) */
+            void invalidate(queryKeys.agent);
+          }
+        },
+        onEntries: (e: OmniEntry[]) => {
+          entriesNow.current = e;
+          setEntries(e);
+          saveSoon();
+        },
+        /* a corrected request comes with the cards it replaced: they leave the screen */
+        onPending: (a: AgentPendingAction) => {
+          for (const id of a.replaces ?? []) {
+            const old = record.current?.actions.get(id);
+            if (old) track(old, "canceled");
+          }
+          track(a, "staged");
+          saveSoon();
+          setPending(p => [...p.filter(x => x.id !== a.id && !a.replaces?.includes(x.id)), a]);
+          /* the screen follows the request: the thumbnails, the publication, the content's Omnichat screen… */
+          show(a.open);
+        },
+        /* the user said yes or no out loud and Omni pressed the card's button (confirm_action) */
+        onResolved: (r: AgentResolvedAction) => {
+          const a = pendingNow.current.find(x => x.id === r.id) ?? r;
+          track(a, r.state);
+          saveSoon();
+          /* a failed publish stays, to be retried, like after the screen's Confirmar */
+          if (r.state !== "failed") setPending(p => p.filter(x => x.id !== r.id));
+          show(r.open);
+          if (r.state === "confirmed") afterConfirm(a);
+        },
+        onError: setError,
+        onQuota: () => {
+          void invalidate(queryKeys.agent);
+          onQuota.current?.();
+        },
+      };
+      setMode(mode);
+      const c = mode === "grok" ? new OmniConversation(callbacks, history) : new OmniCascade(callbacks, mode, history);
+      conv.current = c;
+      analyser.current = c.analyser;
+      micAnalyser.current = c.micAnalyser;
+      if (opts.holdIdle) c.holdIdle(true);
+      const started = c.start();
+      if (opts.greet) void c.greet(opts.greet);
+      await started;
+      if (conv.current === c && c.sessionId) setStartedAt(Date.now());
+    },
+    [],
+  );
+
+  const stop = useCallback(() => {
+    conv.current?.stop();
+    conv.current = null;
+  }, []);
+
+  /* the call the panel opened by itself, ended because nobody spoke: it leaves nothing on screen */
+  const discard = useCallback(() => {
+    conv.current?.stop("idle");
+    conv.current = null;
+    entriesNow.current = [];
+    setEntries([]);
+    setError(null);
+  }, []);
+
+  const holdIdle = useCallback((hold: boolean) => conv.current?.holdIdle(hold), []);
+
+  const setMuted = useCallback((m: boolean) => {
+    conv.current?.setMuted(m);
+    setMutedState(m);
+  }, []);
+
+  const resolve = useCallback(async (action: AgentPendingAction, confirm: boolean) => {
+    const c = conv.current;
+    /* the card belongs to a live conversation (hanging up cancels it); without one, straight to the API */
+    const r = c
+      ? confirm
+        ? await c.confirm(action)
+        : (await c.cancel(action), { ok: true })
+      : await send(
+          confirm
+            ? api.agent.actions[":id"].confirm.$post({ param: { id: enc(action.id) }, json: {} })
+            : api.agent.actions[":id"].cancel.$post({ param: { id: enc(action.id) }, json: {} }),
+        );
+    track(action, !confirm ? "canceled" : r.ok ? "confirmed" : "failed");
+    saveSoon();
+    if (r.ok) setPending(p => p.filter(x => x.id !== action.id));
+    if (r.ok && confirm) {
+      show((r as { open?: NavigateArgs | null }).open);
+      afterConfirm(action);
+    }
+    return r;
+  }, []);
+
+  useEffect(() => () => conv.current?.stop("left"), []);
+
+  return {
+    status,
+    entries,
+    pending,
+    error,
+    muted,
+    startedAt,
+    mode,
+    analyser,
+    micAnalyser,
+    start,
+    stop,
+    setMuted,
+    discard,
+    holdIdle,
+    sendText: (t: string) => conv.current?.sendText(t),
+    feedback: (item: string, v: "up" | "down" | null) => conv.current?.feedback(item, v),
+    resolve,
+  };
+}

package/studio/web/hooks/useOmniClaude.ts

@@ -0,0 +1,124 @@
+/* Omni in Claude mode (after the day's voice quota): the user speaks (or types), the speech is
+   transcribed (/api/agent/stt) and goes to the user's own Claude (/api/agent/claude, a studio
+   session of type "omni"); the answer comes back as text in the widget. */
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import { api, enc, read, send } from "../lib/api";
+import type { OmniEntry } from "../lib/omni-realtime";
+import { PcmRecorder, pcmToBase64 } from "../lib/pcm-recorder";
+
+export type ClaudeStatus = "idle" | "recording" | "transcribing" | "thinking";
+
+const POLL_MS = 1500;
+
+/* the panel shows plain text: markdown marks Claude may still use are dropped */
+const plain = (text: string | null) =>
+  (text ?? "")
+    .replace(/\*\*|__|`/g, "")
+    .replace(/^#+\s*/gm, "")
+    .trim();
+const MAX_WAIT_MS = 5 * 60_000;
+
+export function useOmniClaude() {
+  const [status, setStatus] = useState<ClaudeStatus>("idle");
+  const [entries, setEntries] = useState<OmniEntry[]>([]);
+  const [error, setError] = useState<string | null>(null);
+  const recorder = useRef<PcmRecorder | null>(null);
+  const session = useRef<string | null>(null);
+  const analyser = useRef<AnalyserNode | null>(null);
+  const alive = useRef(true);
+  useEffect(
+    () => () => {
+      alive.current = false;
+      recorder.current?.cancel();
+    },
+    [],
+  );
+
+  const push = useCallback((e: OmniEntry) => setEntries(list => [...list, e]), []);
+
+  const ask = useCallback(
+    async (text: string) => {
+      push({ kind: "user", id: `u-${Date.now()}`, text, pending: false });
+      setStatus("thinking");
+      const t0 = performance.now();
+      const r = await send(api.agent.claude.$post({ json: { message: text } }));
+      if (!r.ok || !r.session) {
+        setError(r.error ?? "não deu para falar com o Claude");
+        setStatus("idle");
+        return;
+      }
+      session.current = r.session;
+      const id = r.session;
+      while (alive.current && performance.now() - t0 < MAX_WAIT_MS) {
+        await new Promise(res => setTimeout(res, POLL_MS));
+        const s = await read(api.agent.claude[":id"].$get({ param: { id: enc(id) } })).catch(() => null);
+        if (!s || s.state === "running") continue;
+        push({
+          kind: "assistant",
+          id: `c-${id}-${Date.now()}`,
+          text: plain(s.text) || "(o Claude terminou sem resposta em texto)",
+          latency_ms: Math.round(performance.now() - t0),
+          feedback: null,
+        });
+        break;
+      }
+      if (alive.current) setStatus("idle");
+    },
+    [push],
+  );
+
+  /* tap: start listening; tap again (or a pause) sends */
+  const toggleRecording = useCallback(async () => {
+    if (recorder.current) return recorder.current.stop();
+    setError(null);
+    const rec = new PcmRecorder();
+    recorder.current = rec;
+    analyser.current = rec.analyser;
+    setStatus("recording");
+    try {
+      const pcm = await rec.record();
+      recorder.current = null;
+      if (!pcm) {
+        setStatus("idle");
+        return;
+      }
+      setStatus("transcribing");
+      const r = await send(
+        api.agent.stt.$post({ json: { session: session.current ?? "claude", audio: pcmToBase64(pcm) } }),
+      );
+      if (!r.ok || !("text" in r) || !r.text) {
+        setError(r.error ?? "não entendi o áudio");
+        setStatus("idle");
+        return;
+      }
+      await ask(r.text);
+    } catch (e) {
+      recorder.current = null;
+      setError(String((e as Error)?.message ?? e));
+      setStatus("idle");
+    }
+  }, [ask]);
+
+  const sendText = useCallback(
+    (text: string) => {
+      const t = text.trim();
+      if (t) void ask(t);
+    },
+    [ask],
+  );
+
+  const feedback = useCallback((item: string, value: "up" | "down" | null) => {
+    setEntries(list => list.map(e => (e.kind === "assistant" && e.id === item ? { ...e, feedback: value } : e)));
+    const id = session.current;
+    if (id)
+      void send(
+        api.agent.sessions[":id"].log.$post({
+          param: { id: enc(id) },
+          json: { type: "feedback", item, value: value ?? "none" },
+        }),
+      );
+  }, []);
+
+  return { status, entries, error, analyser, toggleRecording, sendText, feedback };
+}