@crievo/omni 0.0.0-stage → 0.1.0

package/studio/api/services/agent-cascade.ts

@@ -0,0 +1,975 @@
+/* Omni in cascade: speech → text (xAI STT) → a fast LLM with Omni's tools (OpenRouter) → speech
+   (Fish Audio, the free model first). The cheap alternative to Grok Voice, which bills every second
+   the session is open: here only speech is billed (STT by the second of audio, the LLM by token,
+   Fish free under fair use, the paid model only when the free one fails).
+
+   Two ways to hear the user, picked in the widget:
+   - "stream": the browser streams the microphone to xAI's STT WebSocket (wss://api.x.ai/v1/stt, with
+     an ephemeral token minted here); Smart Turn says when the user finished. $0.20 per hour streamed.
+   - "chunks": the browser records each utterance (its own pause detection) and posts it to
+     /api/agent/stt (REST, $0.10 per hour of speech only).
+   Both post the text to runTurn, which streams NDJSON back: the reply's text, the tools called, the
+   staged actions and the audio (PCM16 mono 24 kHz), sentence by sentence, so the first sentence
+   plays while the LLM is still writing the rest. The tools are the same as Grok's (agent-tools.ts)
+   and run here, with the same guardrails: actions only stage a card for "Confirmar".
+
+   Config `agent.cascade` in omnivideo.config.json (all optional):
+   { llm, llm_fallback, fish_model, fish_fallback, voice, speed, voices: { female, male },
+     stt: { smart_turn, smart_turn_timeout_ms, endpointing_ms }, chunk_pause_ms }. */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { ROOT, readJson } from "./root";
+import { readEnvKeys } from "./services";
+import {
+  AGENT_DIR,
+  OMNI_OFF,
+  agentConfig,
+  engineMismatch,
+  logEvent,
+  logResolved,
+  sessionContext,
+  spentToday,
+} from "./agent";
+import { omniPersona } from "./agent-persona";
+import {
+  cancelSessionActions,
+  isActionTool,
+  isLoggedTool,
+  noteUserTurn,
+  registerToolSession,
+  runTool,
+  TOOL_DEFS,
+} from "./agent-tools";
+import { checkUiCall, isUiTool, UI_TOOL_DEFS } from "./agent-ui-tools";
+import { WAIT_TOOL_NAME } from "./agent-wait";
+import { FAREWELL_REPLY, isFarewell } from "../../shared/omni-farewell";
+import { describePath } from "../../shared/omni-screens";
+import type { CascadeMode } from "../../shared/schemas/agent-cascade";
+
+const XAI = "https://api.x.ai/v1";
+const OPENROUTER = "https://openrouter.ai/api/v1/chat/completions";
+const FISH = "https://api.fish.audio/v1/tts";
+/* Fish paid models: $15 per million UTF-8 bytes of text */
+const FISH_PAID_PER_BYTE = 15 / 1e6;
+const RATE = 24000;
+
+export type CascadeConfig = {
+  llm: string;
+  llm_fallback: string;
+  fish_model: string;
+  fish_fallback: string;
+  /* one Fish voice for Omni whatever agent.gender says; without it, voices[gender] */
+  voice: string | null;
+  /* Fish prosody speed (1 = normal) */
+  speed: number;
+  voices: { female: string; male: string };
+  stt: { smart_turn: number; smart_turn_timeout_ms: number; endpointing_ms: number };
+  chunk_pause_ms: number;
+  /* silence: Omni asks "Ainda tá aí?" at nudge_s of quiet and says goodbye and hangs up at close_s
+     (both counted from when the silence began, like the widget's 6 s / 10 s for Grok) */
+  idle: { nudge_s: number; close_s: number };
+};
+/* voices from Fish's public library, chosen for conversation in pt-BR and not imitating anyone:
+   "Mulher comum" and "Linn" */
+const DEFAULTS: CascadeConfig = {
+  llm: "anthropic/claude-haiku-5.5",
+  llm_fallback: "google/gemini-3.1-flash-lite",
+  fish_model: "s2.1-pro-free",
+  fish_fallback: "s2.1-pro",
+  voice: null,
+  speed: 1,
+  voices: { female: "148da69cf74b4530b1c41d5c1e87c688", male: "c7d997eba056434d8ba3dd1b8dec560e" },
+  /* measured: 0.5 / 1200 / 300 took ~0.9 s from the end of speech to the text; this ends turns sooner */
+  stt: { smart_turn: 0.3, smart_turn_timeout_ms: 600, endpointing_ms: 200 },
+  chunk_pause_ms: 700,
+  idle: { nudge_s: 6, close_s: 10 },
+};
+
+const str = (v: unknown, fallback: string) => (typeof v === "string" && v.trim() ? v.trim() : fallback);
+const num = (v: unknown, fallback: number, min: number, max: number) =>
+  typeof v === "number" && Number.isFinite(v) && v >= min && v <= max ? v : fallback;
+
+export function cascadeConfig(): CascadeConfig {
+  const c = readJson<any>(join(ROOT, "omnivideo.config.json"))?.agent?.cascade ?? {};
+  const d = DEFAULTS;
+  return {
+    llm: str(c.llm, d.llm),
+    llm_fallback: str(c.llm_fallback, d.llm_fallback),
+    fish_model: str(c.fish_model, d.fish_model),
+    fish_fallback: str(c.fish_fallback, d.fish_fallback),
+    voice: typeof c.voice === "string" && /^[0-9a-f]{32}$/.test(c.voice) ? c.voice : d.voice,
+    speed: num(c.speed, d.speed, 0.5, 2),
+    voices: { female: str(c.voices?.female, d.voices.female), male: str(c.voices?.male, d.voices.male) },
+    stt: {
+      smart_turn: num(c.stt?.smart_turn, d.stt.smart_turn, 0, 1),
+      smart_turn_timeout_ms: num(c.stt?.smart_turn_timeout_ms, d.stt.smart_turn_timeout_ms, 1, 5000),
+      endpointing_ms: num(c.stt?.endpointing_ms, d.stt.endpointing_ms, 0, 5000),
+    },
+    chunk_pause_ms: num(c.chunk_pause_ms, d.chunk_pause_ms, 300, 3000),
+    idle: {
+      nudge_s: num(c.idle?.nudge_s, d.idle.nudge_s, 2, 120),
+      close_s: num(c.idle?.close_s, d.idle.close_s, 3, 180),
+    },
+  };
+}
+
+function key(name: string): string {
+  const k = readEnvKeys()[name];
+  if (!k) throw new Error(`falta ${name} no .env`);
+  return k;
+}
+
+/* --- session ------------------------------------------------------------------------------------ */
+/* who Omni is, what she does and doesn't do (agent-persona.ts, shared with Grok), in the grammar of
+   the voice the admin picked (agent.gender) */
+
+/* how the reply should sound: it becomes voice, so no markdown, lists or emoji */
+const VOICE_RULES = `Sua resposta vira voz na hora:
+- Responda como numa conversa: no máximo duas frases curtas, uns 35 palavras. Se houver mais a dizer, dê o principal e pergunte se a pessoa quer o resto.
+- Nada de markdown, listas, emojis, links ou slugs.
+- Escreva números como se fala ("uns três mil views", "quarenta por cento").
+- Resposta que não precisa de ferramenta: comece com uma frase de duas a quatro palavras, terminada em ponto, que já vira voz enquanto você pensa no resto ("Claro, vamos lá.", "Hum, boa.", "Certo."). Varie, não repita a mesma da resposta anterior.
+- Quando precisar de ferramenta (ver dados, abrir uma tela, preparar ou mudar algo), chame direto, sem escrever nada antes: o estúdio já diz ao usuário, na hora, que você vai checar, abrir ou preparar.
+- Depois do resultado, responda direto o que foi pedido. Nada de "deixa eu ver", "vou checar", "só um minuto": o usuário já ouviu o aviso e você já checou.
+- Mensagens que começam com [Tela] contam o que o usuário fez na tela (Confirmar, Cancelar): reaja em uma frase.
+- [Tela atual: …] diz onde o usuário está: "esse vídeo", "essa cena" e "aqui" são o que está nessa tela.`;
+
+type Msg = Record<string, unknown>;
+type CascadeSession = {
+  id: string;
+  mode: CascadeMode;
+  started: number;
+  voice: string;
+  messages: Msg[];
+  /* a turn in flight is aborted when the next one starts (the user spoke over Omni) */
+  turn: AbortController | null;
+  /* the speculative turn in flight, until the page confirms or drops it */
+  spec: Speculation | null;
+  /* UI tool calls waiting for the browser's result, by call id */
+  ui: Map<string, (output: Record<string, unknown>) => void>;
+  /* the screen the last turn said the user was on */
+  screen: string | null;
+  /* the tool notices played so far: the next one picks another phrase */
+  acks: number;
+  /* UI tools that timed out in a row, by name: after UI_TIMEOUT_LIMIT the call is refused without waiting */
+  uiTimeouts: Map<string, { n: number; at: number }>;
+};
+const sessions = new Map<string, CascadeSession>();
+
+export type CascadeStart = {
+  id: string;
+  mode: CascadeMode;
+  config: ReturnType<typeof agentConfig>;
+  chunk_pause_ms: number;
+  idle: CascadeConfig["idle"];
+  /* stream mode: where the browser opens the STT WebSocket, and the token for it */
+  stt: { url: string; token: string } | null;
+};
+
+/* `history`: an earlier conversation the user continues (the Omni panel's Conversas): its lines go
+   back into `messages` as they were said, after the persona and today's context */
+export async function startCascade(
+  mode: CascadeMode,
+  history: { role: "user" | "assistant"; text: string }[] = [],
+): Promise<CascadeStart> {
+  const config = agentConfig();
+  if (!config.enabled) throw new Error(OMNI_OFF);
+  if (config.engine !== mode) throw new Error(engineMismatch(config.engine));
+  const spent = spentToday();
+  if (spent >= config.limits.daily_usd)
+    throw new Error(
+      `limite diário do Omni atingido: US$ ${spent.toFixed(2)} de US$ ${config.limits.daily_usd.toFixed(2)} (agent.limits.daily_usd)`,
+    );
+  key("OPENROUTER_API_KEY");
+  key("FISH_API_KEY");
+  const cc = cascadeConfig();
+  const now = Date.now();
+  for (const [k, s] of sessions) if (now - s.started > 6 * 3600_000) sessions.delete(k);
+  const id = `${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "")}-${crypto.randomUUID().slice(0, 6)}`;
+  const voice = cc.voice ?? cc.voices[config.gender];
+  const system = `${omniPersona(config.gender)}\n\n${VOICE_RULES}\n\n${await sessionContext(new Date())}`;
+  sessions.set(id, {
+    id,
+    mode,
+    started: now,
+    voice,
+    /* cache_control: the persona and the tools are the same every turn (Anthropic caches them) */
+    messages: [
+      { role: "system", content: [{ type: "text", text: system, cache_control: { type: "ephemeral" } }] },
+      ...(history.length
+        ? [
+            {
+              role: "system",
+              content:
+                "A conversa abaixo é uma conversa anterior que o usuário pediu para continuar. O estado do estúdio pode ter mudado desde então: confira com as ferramentas antes de afirmar algo.",
+            },
+            ...history.map(h => ({ role: h.role, content: h.text })),
+          ]
+        : []),
+    ],
+    turn: null,
+    spec: null,
+    ui: new Map(),
+    uiTimeouts: new Map(),
+    screen: null,
+    acks: 0,
+  });
+  registerToolSession(id, { max_actions: config.limits.max_actions, max_tool_calls: config.limits.max_tool_calls });
+  logEvent({ session: id, type: "start", mode, voice, llm: cc.llm, tts: cc.fish_model });
+  /* the silence and notice clips, made once per voice and speed, one at a time (the free model is
+     rate limited); the first request also warms Fish up */
+  if (Object.keys(CLIPS).some(k => !existsSync(clipFile(voice, cc, k as ClipName)))) warmFish(voice, cc.fish_model);
+  void (async () => {
+    for (const k of Object.keys(CLIPS)) await cascadeClip(id, k).catch(() => {});
+  })();
+  return {
+    id,
+    mode,
+    config,
+    chunk_pause_ms: cc.chunk_pause_ms,
+    idle: cc.idle,
+    stt: mode === "stream" ? await sttSocket(cc) : null,
+  };
+}
+
+/* the first Fish request of a while pays the TLS handshake and a cold start (~3 s measured):
+   one tiny free request when the session opens, so the first reply doesn't */
+function warmFish(voice: string, model: string) {
+  if (!model.endsWith("-free")) return;
+  void fetch(FISH, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${key("FISH_API_KEY")}`, model, "Content-Type": "application/json" },
+    body: JSON.stringify({ text: "Oi.", reference_id: voice, format: "pcm", sample_rate: RATE, latency: "low" }),
+    signal: AbortSignal.timeout(15_000),
+  })
+    .then(r => r.arrayBuffer())
+    .catch(() => {});
+}
+
+/* what Omni says on its own, in its voice: when nobody speaks (agent.cascade.idle), and the notice
+   that plays the moment the model calls a tool in a turn that said nothing yet (ackFor), so the user
+   hears "Vou checar." before the wait, not after it. Synthesized once and kept in .agent/samples, as
+   raw PCM16 24 kHz */
+const CLIPS = {
+  nudge: "Ainda tá aí?",
+  bye: "Vou encerrar por aqui. É só me chamar quando quiser.",
+  ack_check_1: "Vou checar.",
+  ack_check_2: "Deixa eu ver.",
+  ack_check_3: "Vou conferir.",
+  ack_open_1: "Vou abrir.",
+  ack_open_2: "Já abro.",
+  ack_prepare_1: "Vou preparar.",
+  ack_prepare_2: "Já preparo.",
+  ack_confirm: "Confirmando.",
+  ack_cancel: "Cancelando.",
+  ack_config: "Vou configurar.",
+  ack_write: "Vou escrever.",
+  ack_send: "Vou enviar.",
+  ack_stop: "Vou parar.",
+  ack_search: "Vou buscar.",
+  /* the user said goodbye (shared/omni-farewell) */
+  farewell: FAREWELL_REPLY,
+} as const;
+type ClipName = keyof typeof CLIPS;
+export const isClip = (name: string): name is ClipName => name in CLIPS;
+
+const clipFile = (voice: string, cc: CascadeConfig, name: ClipName) =>
+  join(AGENT_DIR, "samples", `fish-${voice}-${cc.speed}-${name}.pcm`);
+
+/* the notice for a tool, varied between turns; null: the tool needs none (the model speaks for
+   end_call and wait_for_work, and playing or pausing the editor is instant) */
+const ACKS = {
+  check: ["ack_check_1", "ack_check_2", "ack_check_3"],
+  open: ["ack_open_1", "ack_open_2"],
+  prepare: ["ack_prepare_1", "ack_prepare_2"],
+  confirm: ["ack_confirm"],
+  cancel: ["ack_cancel"],
+  config: ["ack_config"],
+  write: ["ack_write"],
+  send: ["ack_send"],
+  stop: ["ack_stop"],
+  search: ["ack_search"],
+} satisfies Record<string, ClipName[]>;
+function ackGroup(tool: string): keyof typeof ACKS | null {
+  if (tool === "end_call" || tool === WAIT_TOOL_NAME || tool === "control_editor") return null;
+  if (tool === "confirm_action") return "confirm";
+  if (tool === "cancel_action") return "cancel";
+  if (tool === "set_omnichat") return "config";
+  if (isActionTool(tool)) return "prepare";
+  if (tool === "navigate" || tool === "open_claude" || tool === "show_card" || tool === "ask_in_chat") return "open";
+  if (tool === "write_to_claude") return "write";
+  if (tool === "send_to_claude") return "send";
+  if (tool === "stop_claude") return "stop";
+  if (tool === "add_epidemic_sound") return "search";
+  return "check";
+}
+export function ackFor(tool: string, turn: number): ClipName | null {
+  const group = ackGroup(tool);
+  if (!group) return null;
+  const options: readonly ClipName[] = ACKS[group];
+  return options[turn % options.length];
+}
+/* a clip already on disk (startCascade makes them all); null while it is still being made */
+function clipNow(voice: string, cc: CascadeConfig, name: ClipName): Uint8Array | null {
+  try {
+    return new Uint8Array(readFileSync(clipFile(voice, cc, name)));
+  } catch {
+    return null;
+  }
+}
+
+/* what a reply that comes after a tool should not open with: the user already heard the notice, and
+   the tool already ran. One short sentence, so a real answer ("Vou checar a agenda: tem dois.") stays */
+const FILLER =
+  /^(?:(?:ok|certo|beleza|claro|pronto|hum+|então|tá)[,.!]?\s+)?(?:(?:(?:deixa|deixe) eu|deixe-me|vou|já vou|só vou|vamos)\s+(?:dar uma olhada|ver|checar|conferir|olhar|verificar|buscar|consultar|pesquisar)|(?:só )?(?:um|uma) (?:minuto|minutinho|momento|segundo|segundinho|instante)|(?:checando|verificando|conferindo|olhando|buscando))\b[^.!?…:;]{0,40}[.!?…]+(?:\s+|$)/i;
+/* the leading filler sentences of a reply, removed; `done`: the text may still grow (a sentence
+   without its end yet is kept until it ends or grows past a short sentence) */
+export function stripFiller(text: string, done: boolean): { text: string; settled: boolean } {
+  let t = text.trimStart();
+  for (let m = FILLER.exec(t); m; m = FILLER.exec(t)) t = t.slice(m[0].length);
+  const settled = done || /[.!?…:;](\s|$)/.test(t) || t.length > 80;
+  return { text: t, settled };
+}
+
+const making = new Map<string, Promise<Uint8Array>>();
+
+export async function cascadeClip(id: string, name: string): Promise<Uint8Array> {
+  const s = sessions.get(id);
+  if (!s || !isClip(name)) throw new Error("trecho inválido");
+  const cc = cascadeConfig();
+  const file = clipFile(s.voice, cc, name);
+  if (existsSync(file)) return new Uint8Array(readFileSync(file));
+  const pending = making.get(file);
+  if (pending) return pending;
+  const job = (async () => {
+    for (const model of [cc.fish_model, cc.fish_fallback]) {
+      const r = await fetch(FISH, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${key("FISH_API_KEY")}`, model, "Content-Type": "application/json" },
+        body: JSON.stringify({
+          text: CLIPS[name],
+          reference_id: s.voice,
+          format: "pcm",
+          sample_rate: RATE,
+          latency: "normal",
+          prosody: { speed: cc.speed, volume: 0 },
+        }),
+        signal: AbortSignal.timeout(20_000),
+      }).catch(() => null);
+      if (!r?.ok) continue;
+      const pcm = new Uint8Array(await r.arrayBuffer());
+      mkdirSync(join(AGENT_DIR, "samples"), { recursive: true });
+      writeFileSync(file, pcm);
+      return pcm;
+    }
+    throw new Error("o Fish não gerou o trecho");
+  })().finally(() => making.delete(file));
+  making.set(file, job);
+  return job;
+}
+
+/* the STT WebSocket for stream mode: the token only opens it (60 s) and the page asks again to reconnect */
+export async function sttSocket(cc = cascadeConfig()): Promise<{ url: string; token: string }> {
+  const r = await fetch(`${XAI}/realtime/client_secrets`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${key("XAI_API_KEY")}`, "Content-Type": "application/json" },
+    body: JSON.stringify({ expires_after: { seconds: 60 } }),
+    signal: AbortSignal.timeout(15_000),
+  });
+  if (!r.ok) throw new Error(`xAI ${r.status}: ${(await r.text()).slice(0, 200)}`);
+  const secret: any = await r.json();
+  const q = new URLSearchParams({
+    model: "grok-voice-transcribe-2.0",
+    sample_rate: String(RATE),
+    encoding: "pcm",
+    interim_results: "true",
+    language: agentConfig().language.slice(0, 2),
+    smart_turn: String(cc.stt.smart_turn),
+    smart_turn_timeout: String(cc.stt.smart_turn_timeout_ms),
+    endpointing: String(cc.stt.endpointing_ms),
+  });
+  return { url: `wss://api.x.ai/v1/stt?${q}`, token: secret.value };
+}
+
+export const hasCascade = (id: string) => sessions.has(id);
+
+/* streamed STT seconds, reported by the page when the conversation ends: $0.20 per hour */
+export function endCascade(id: string, durationS: number, sttAudioS: number | undefined, reason?: string) {
+  const s = sessions.get(id);
+  const wall = s ? (Date.now() - s.started) / 1000 : durationS;
+  const streamed = s?.mode === "stream" ? Math.min(sttAudioS ?? durationS, wall + 5) : 0;
+  logEvent({ session: id, type: "end", duration_s: durationS, reason, mode: s?.mode });
+  if (streamed > 0)
+    logEvent({
+      session: id,
+      type: "stt",
+      mode: "stream",
+      duration_s: Math.round(streamed * 10) / 10,
+      cost_usd: (streamed * 0.2) / 3600,
+    });
+  for (const a of cancelSessionActions(id))
+    logEvent({ session: id, type: "action", kind: a.kind, slug: a.slug, canceled: true, reason: "hangup" });
+  s?.turn?.abort();
+  sessions.delete(id);
+}
+
+/* --- a turn ------------------------------------------------------------------------------------- */
+export type TurnEvent =
+  | { type: "text"; delta: string }
+  | { type: "tool"; id: string; name: string; state: "running" | "ok" | "error"; ms?: number; detail?: string | null }
+  | { type: "pending"; action: unknown }
+  /* a card decided by voice (confirm_action, cancel_action): the page takes it away and opens its screen */
+  | { type: "resolved"; resolved: unknown }
+  /* a tool that changed something by itself (set_omnichat): the page opens the screen that shows it */
+  | { type: "open"; open: unknown }
+  | { type: "audio"; pcm: string }
+  | { type: "end_call" }
+  | { type: "ui_call"; id: string; name: string; args: unknown }
+  | {
+      type: "done";
+      llm_first_ms: number | null;
+      tts_first_ms: number | null;
+      ms: number;
+      cost_usd: number;
+      model: string;
+      tts_model: string | null;
+    }
+  | { type: "error"; message: string };
+
+/* the whole registry: the server tools, the UI tools (run in the browser) and OpenRouter's web search
+   (run by OpenRouter inside the call: the stream pauses while it searches, ~1.3 s measured) */
+const TOOLS = [
+  ...[...TOOL_DEFS, ...UI_TOOL_DEFS].map(t => ({
+    type: "function",
+    function: { name: t.name, description: t.description, parameters: t.parameters },
+  })),
+  {
+    type: "openrouter:web_search",
+    parameters: { engine: "parallel", mode: "turbo", max_results: 5, max_uses: 2, search_context_size: "low" },
+  },
+];
+/* how long the browser has to run a UI tool and answer (a Claude tool may navigate, open the panel
+   and wait for the reply box to come back after a stop: up to ~6 s in lib/omni-ui) */
+const UI_TIMEOUT_MS = 9000;
+/* a UI tool that timed out this many times in a row is not called again for UI_TIMEOUT_PAUSE_MS: the
+   page is not answering, and each retry costs the user UI_TIMEOUT_MS of silence (measured: 7 in a row, 63 s) */
+const UI_TIMEOUT_LIMIT = 2;
+const UI_TIMEOUT_PAUSE_MS = 60_000;
+/* a tool's output as the LLM sees it: bounded (get_video can be long) */
+const MAX_TOOL_CHARS = 16_000;
+const MAX_ROUNDS = 5;
+
+type ToolCall = { id: string; name: string; args: string };
+
+/* one LLM call, streamed: text deltas go to onText as they arrive; the tool calls come back whole,
+   and onCall hears each one's name as soon as it starts (its arguments still streaming) */
+async function complete(
+  model: string,
+  messages: Msg[],
+  signal: AbortSignal,
+  onText: (delta: string) => void,
+  onCall: (name: string) => void = () => {},
+): Promise<{ text: string; calls: ToolCall[]; cost: number; firstMs: number | null }> {
+  const t0 = performance.now();
+  const r = await fetch(OPENROUTER, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${key("OPENROUTER_API_KEY")}`, "Content-Type": "application/json" },
+    /* no reasoning: with it Haiku took 5–6 s after a tool to say one sentence (the tools precompute
+       what it would reason about, like published_counts) */
+    body: JSON.stringify({
+      model,
+      messages,
+      tools: TOOLS,
+      stream: true,
+      max_tokens: 500,
+      reasoning: { effort: "none" },
+      usage: { include: true },
+    }),
+    signal,
+  });
+  if (!r.ok || !r.body) throw new Error(`${model} ${r.status}: ${(await r.text()).slice(0, 200)}`);
+  const reader = r.body.getReader();
+  const dec = new TextDecoder();
+  let buf = "";
+  let text = "";
+  let cost = 0;
+  let firstMs: number | null = null;
+  const calls = new Map<number, ToolCall>();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buf += dec.decode(value, { stream: true });
+    let nl = buf.indexOf("\n");
+    while (nl >= 0) {
+      const line = buf.slice(0, nl).trim();
+      buf = buf.slice(nl + 1);
+      nl = buf.indexOf("\n");
+      if (!line.startsWith("data:") || line === "data: [DONE]") continue;
+      let j: any;
+      try {
+        j = JSON.parse(line.slice(5));
+      } catch {
+        continue;
+      }
+      if (j.error) throw new Error(`${model}: ${j.error.message ?? "erro"}`);
+      const d = j.choices?.[0]?.delta;
+      if (d?.content) {
+        firstMs ??= Math.round(performance.now() - t0);
+        text += d.content;
+        onText(d.content);
+      }
+      for (const tc of d?.tool_calls ?? []) {
+        firstMs ??= Math.round(performance.now() - t0);
+        const c = calls.get(tc.index ?? 0) ?? { id: "", name: "", args: "" };
+        if (tc.id) c.id = tc.id;
+        if (tc.function?.name) {
+          const first = !c.name;
+          c.name += tc.function.name;
+          if (first) onCall(c.name);
+        }
+        if (tc.function?.arguments) c.args += tc.function.arguments;
+        calls.set(tc.index ?? 0, c);
+      }
+      if (j.usage) cost = Number(j.usage.cost) || 0;
+    }
+  }
+  return {
+    text,
+    calls: [...calls.values()].map((c, i) => ({ ...c, id: c.id || `call_${i}` })),
+    cost,
+    firstMs,
+  };
+}
+
+/* the reply's text → sentences → Fish, in order: each sentence's request starts as soon as the
+   sentence is complete, and its audio is emitted only after the previous one's */
+class Speaker {
+  firstMs: number | null = null;
+  model: string | null = null;
+  paidBytes = 0;
+  private buf = "";
+  private spoken = 0;
+  private chain: Promise<void> = Promise.resolve();
+
+  constructor(
+    private voice: string,
+    private cc: CascadeConfig,
+    private t0: number,
+    private signal: AbortSignal,
+    private emit: (e: TurnEvent) => void,
+  ) {}
+
+  push(delta: string) {
+    this.buf += delta;
+    for (;;) {
+      const cut = this.boundary();
+      if (cut < 0) break;
+      this.say(this.buf.slice(0, cut));
+      this.buf = this.buf.slice(cut);
+    }
+  }
+
+  /* the end of the next sentence; the first piece may also break at a comma (or, past 50
+     characters, at a space), so the first audio comes sooner */
+  private boundary(): number {
+    const m = /[.!?…:;](\s|$)|\n/.exec(this.buf);
+    if (m && (m.index > 0 || this.spoken > 0)) return m.index + m[0].length;
+    if (this.spoken === 0) {
+      const comma = this.buf.indexOf(", ", 15);
+      if (comma > 0) return comma + 2;
+      if (this.buf.length > 50) {
+        const space = this.buf.lastIndexOf(" ");
+        if (space > 30) return space + 1;
+      }
+    }
+    return -1;
+  }
+
+  flush() {
+    if (this.buf.trim()) this.say(this.buf);
+    this.buf = "";
+  }
+
+  /* audio that is already made (a clip), in its turn after what was said before it */
+  pcm(bytes: Uint8Array) {
+    if (this.signal.aborted) return;
+    this.spoken++;
+    this.chain = this.chain
+      .then(() => {
+        for (let i = 0; i < bytes.length && !this.signal.aborted; i += 24_000) {
+          this.firstMs ??= Math.round(performance.now() - this.t0);
+          this.emit({ type: "audio", pcm: Buffer.from(bytes.subarray(i, i + 24_000)).toString("base64") });
+        }
+      })
+      .catch(() => {});
+  }
+
+  private say(sentence: string) {
+    /* markdown the model may still write is not read aloud */
+    const text = sentence
+      .replace(/[*_#`>]/g, "")
+      .replace(/^\s*[-•]\s+/gm, "")
+      .trim();
+    if (!text || this.signal.aborted) return;
+    this.spoken++;
+    const audio = this.request(text);
+    this.chain = this.chain.then(() => this.drain(audio)).catch(() => {});
+  }
+
+  private async request(text: string): Promise<Response | null> {
+    for (const model of [this.cc.fish_model, this.cc.fish_fallback]) {
+      try {
+        const r = await fetch(FISH, {
+          method: "POST",
+          headers: { Authorization: `Bearer ${key("FISH_API_KEY")}`, model, "Content-Type": "application/json" },
+          body: JSON.stringify({
+            text,
+            reference_id: this.voice,
+            format: "pcm",
+            sample_rate: RATE,
+            latency: "low",
+            prosody: { speed: this.cc.speed, volume: 0 },
+          }),
+          signal: this.signal,
+        });
+        if (r.ok && r.body) {
+          this.model = model;
+          if (!model.endsWith("-free")) this.paidBytes += new TextEncoder().encode(text).length;
+          return r;
+        }
+      } catch {
+        if (this.signal.aborted) return null;
+      }
+    }
+    return null;
+  }
+
+  private async drain(audio: Promise<Response | null>) {
+    const r = await audio;
+    if (!r?.body || this.signal.aborted) return;
+    const reader = r.body.getReader();
+    let carry: Uint8Array | null = null;
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done || this.signal.aborted) break;
+      let bytes = value;
+      if (carry) {
+        bytes = new Uint8Array(carry.length + value.length);
+        bytes.set(carry);
+        bytes.set(value, carry.length);
+        carry = null;
+      }
+      /* PCM16: whole samples only */
+      if (bytes.length % 2) {
+        carry = bytes.slice(-1);
+        bytes = bytes.slice(0, -1);
+      }
+      if (!bytes.length) continue;
+      this.firstMs ??= Math.round(performance.now() - this.t0);
+      this.emit({ type: "audio", pcm: Buffer.from(bytes).toString("base64") });
+    }
+  }
+
+  done() {
+    return this.chain;
+  }
+}
+
+/* a speculative turn starts on the STT's locked text, before it says the user finished: the page
+   holds its events until it confirms (commitTurn) and drops it if the user goes on talking. Until
+   then it runs no tool, and dropped, it leaves nothing in the history. */
+type Speculation = { committed: boolean; wake: ((go: boolean) => void) | null };
+
+/* a UI tool runs in the browser: the call goes down the turn's stream and the browser posts the
+   result back (resolveUiCall); no answer in time, or the turn cut, and the model hears why */
+async function uiCall(
+  s: CascadeSession,
+  c: ToolCall,
+  emit: (e: TurnEvent) => void,
+  signal: AbortSignal,
+): Promise<Record<string, unknown>> {
+  const invalid = checkUiCall(c.name, c.args);
+  if (invalid) return { error: invalid };
+  const failed = s.uiTimeouts.get(c.name);
+  if (failed && Date.now() - failed.at > UI_TIMEOUT_PAUSE_MS) s.uiTimeouts.delete(c.name);
+  else if (failed && failed.n >= UI_TIMEOUT_LIMIT)
+    return {
+      error:
+        "a tela não está respondendo a essa ferramenta: não tente de novo, diga ao usuário e siga por outro caminho",
+    };
+  let args: unknown;
+  try {
+    args = JSON.parse(c.args || "{}");
+  } catch {
+    return { error: "argumentos não são JSON" };
+  }
+  return new Promise(resolve => {
+    const done = (output: Record<string, unknown>) => {
+      clearTimeout(timer);
+      s.ui.delete(c.id);
+      if (output.error === "a tela não respondeu")
+        s.uiTimeouts.set(c.name, { n: (s.uiTimeouts.get(c.name)?.n ?? 0) + 1, at: Date.now() });
+      else if (output.error !== "interrompido") s.uiTimeouts.delete(c.name);
+      resolve(output);
+    };
+    const timer = setTimeout(() => done({ error: "a tela não respondeu" }), UI_TIMEOUT_MS);
+    signal.addEventListener("abort", () => done({ error: "interrompido" }), { once: true });
+    s.ui.set(c.id, done);
+    emit({ type: "ui_call", id: c.id, name: c.name, args });
+  });
+}
+
+export function resolveUiCall(id: string, callId: string, output: Record<string, unknown>): boolean {
+  const done = sessions.get(id)?.ui.get(callId);
+  if (!done) return false;
+  done(output);
+  return true;
+}
+
+export function commitTurn(id: string): boolean {
+  const spec = sessions.get(id)?.spec;
+  if (!spec) return false;
+  spec.committed = true;
+  spec.wake?.(true);
+  return true;
+}
+
+function confirmed(spec: Speculation, signal: AbortSignal): Promise<boolean> {
+  if (spec.committed) return Promise.resolve(true);
+  if (signal.aborted) return Promise.resolve(false);
+  return new Promise(resolve => {
+    spec.wake = resolve;
+    signal.addEventListener("abort", () => resolve(false), { once: true });
+  });
+}
+
+/* tool calls the history has no result for (the turn was cut while they ran) get one, or the next
+   LLM call would be refused */
+function closeCalls(local: Msg[]) {
+  const answered = new Set(local.filter(m => m.role === "tool").map(m => m.tool_call_id));
+  for (const m of [...local])
+    for (const c of (m.tool_calls as { id: string }[] | undefined) ?? [])
+      if (!answered.has(c.id))
+        local.push({ role: "tool", tool_call_id: c.id, content: JSON.stringify({ error: "interrompido" }) });
+}
+
+export async function runTurn(
+  id: string,
+  text: string,
+  signal: AbortSignal,
+  emit: (e: TurnEvent) => void,
+  speculative = false,
+  screen?: string,
+) {
+  const s = sessions.get(id);
+  if (!s) return emit({ type: "error", message: "sessão desconhecida: encerre e abra a conversa de novo" });
+  /* the admin switched Omni off or moved it to another engine: this conversation stops here */
+  const now = agentConfig();
+  if (!now.enabled || now.engine !== s.mode)
+    return emit({ type: "error", message: now.enabled ? engineMismatch(now.engine) : OMNI_OFF });
+  s.turn?.abort();
+  const ctrl = new AbortController();
+  s.turn = ctrl;
+  /* the user's own words (a "[Tela] …" note is the page telling what happened): a card staged before
+     this turn can now be confirmed by voice */
+  if (!text.startsWith("[Tela]")) noteUserTurn(id);
+  const spec: Speculation | null = speculative ? { committed: false, wake: null } : null;
+  s.spec = spec;
+  const abort = () => ctrl.abort();
+  signal.addEventListener("abort", abort);
+  const cc = cascadeConfig();
+  const t0 = performance.now();
+  const speaker = new Speaker(s.voice, cc, t0, ctrl.signal, emit);
+  /* this turn's part of the history: it joins s.messages when the turn ends (or is cut) */
+  /* where the user is, when it changed since the last turn ("esse vídeo" is the one on screen) */
+  const where = screen ? describePath(screen) : null;
+  const note = where && where !== s.screen ? `[Tela atual: ${where}]\n` : "";
+  if (where) s.screen = where;
+  const local: Msg[] = [{ role: "user", content: `${note}${text}` }];
+  const history = () => [...s.messages, ...local];
+  let cost = 0;
+  let llmFirst: number | null = null;
+  let model = cc.llm;
+  /* this round's text so far, as the user heard it (an interrupted round never reaches the history
+     otherwise) */
+  let partial = "";
+  /* whether the user heard anything in this turn yet */
+  let voiced = false;
+  try {
+    /* a goodbye and nothing else: "Até logo!" and the call ends, with no model in between */
+    const bye = !text.startsWith("[Tela]") && isFarewell(text);
+    if (bye) {
+      const bytes = clipNow(s.voice, cc, "farewell");
+      emit({ type: "text", delta: FAREWELL_REPLY });
+      if (bytes) speaker.pcm(bytes);
+      else speaker.push(`${FAREWELL_REPLY} `);
+      speaker.flush();
+      local.push({ role: "assistant", content: FAREWELL_REPLY });
+    }
+    for (let round = 0; !bye && round < MAX_ROUNDS; round++) {
+      partial = "";
+      /* a round after a tool continues the same spoken reply: keep a space between them */
+      let gap = local.some(m => m.role === "assistant" && m.content);
+      const say = (raw: string) => {
+        const d = gap ? ` ${raw.trimStart()}` : raw;
+        gap = false;
+        partial += d;
+        voiced = true;
+        emit({ type: "text", delta: d });
+        speaker.push(d);
+      };
+      /* after the notice (or anything said before a tool), the reply goes straight to the answer: a
+         "Deixa eu ver." it opens with is held until its sentence ends and dropped */
+      let held: string | null = voiced ? "" : null;
+      const release = (done: boolean) => {
+        if (held === null) return;
+        const { text, settled } = stripFiller(held, done);
+        if (!settled) return;
+        held = null;
+        if (text) say(text);
+      };
+      /* whether the model itself wrote anything this round (the notice doesn't count) */
+      let wrote = false;
+      const onText = (raw: string) => {
+        wrote = true;
+        if (held === null) return say(raw);
+        held += raw;
+        release(false);
+      };
+      /* a silent turn that calls a tool: the user hears the notice now, while the tool runs */
+      const onCall = (name: string) => {
+        if (voiced || ctrl.signal.aborted) return;
+        const clip = ackFor(name, s.acks);
+        if (!clip) return;
+        s.acks++;
+        const bytes = clipNow(s.voice, cc, clip);
+        if (bytes) {
+          const d = gap ? ` ${CLIPS[clip]}` : CLIPS[clip];
+          gap = false;
+          partial += d;
+          voiced = true;
+          emit({ type: "text", delta: d });
+          speaker.pcm(bytes);
+        } else say(`${CLIPS[clip]} `);
+        held = "";
+      };
+      let out: Awaited<ReturnType<typeof complete>>;
+      try {
+        out = await complete(model, history(), ctrl.signal, onText, onCall);
+      } catch (e) {
+        /* the fallback only when nothing was said yet: a reply doesn't restart halfway */
+        if (ctrl.signal.aborted || wrote || model === cc.llm_fallback) throw e;
+        model = cc.llm_fallback;
+        out = await complete(model, history(), ctrl.signal, onText, onCall);
+      }
+      cost += out.cost;
+      if (round === 0) llmFirst = out.firstMs;
+      release(true);
+      speaker.flush();
+      /* the history keeps what was said (the notice in, the filler out), so the model doesn't repeat it */
+      const said = partial.trim();
+      partial = "";
+      local.push({
+        role: "assistant",
+        content: said || null,
+        ...(out.calls.length
+          ? {
+              tool_calls: out.calls.map(c => ({
+                id: c.id,
+                type: "function",
+                function: { name: c.name, arguments: c.args || "{}" },
+              })),
+            }
+          : {}),
+      });
+      if (!out.calls.length) break;
+      /* no tool runs on a guess: a speculative turn waits for the user to have really finished */
+      if (spec && !(await confirmed(spec, ctrl.signal))) throw new Error("descartado");
+      let ending = false;
+      for (const c of out.calls) {
+        if (c.name === "end_call") ending = true;
+        const tid = `tool-${c.id}`;
+        if (c.name !== "end_call") emit({ type: "tool", id: tid, name: c.name, state: "running" });
+        const t = performance.now();
+        const output = isUiTool(c.name)
+          ? await uiCall(s, c, emit, ctrl.signal)
+          : await runTool(id, c.name, c.args || "{}");
+        const ms = Math.round(performance.now() - t);
+        const error = typeof output.error === "string" ? output.error : null;
+        logEvent({
+          session: id,
+          type: "tool",
+          name: c.name,
+          ms,
+          error: error ?? undefined,
+          cost_usd: typeof output.cost_usd === "number" ? output.cost_usd : undefined,
+          model: typeof output.model === "string" ? output.model : undefined,
+          args: isLoggedTool(c.name) ? c.args.slice(0, 2000) : undefined,
+        });
+        logResolved(id, output);
+        if (c.name !== "end_call")
+          emit({ type: "tool", id: tid, name: c.name, state: error ? "error" : "ok", ms, detail: error });
+        if (output.pending_action) emit({ type: "pending", action: output.pending_action });
+        if (output.resolved) emit({ type: "resolved", resolved: output.resolved });
+        if (output.open) emit({ type: "open", open: output.open });
+        local.push({ role: "tool", tool_call_id: c.id, content: JSON.stringify(output).slice(0, MAX_TOOL_CHARS) });
+      }
+      if (ending) {
+        await speaker.done();
+        emit({ type: "end_call" });
+        break;
+      }
+    }
+    await speaker.done();
+    /* a guess that finished before the user did: it only counts once confirmed */
+    if (spec && !(await confirmed(spec, ctrl.signal))) throw new Error("descartado");
+    s.messages.push(...local);
+    if (bye) emit({ type: "end_call" });
+    const ttsCost = speaker.paidBytes * FISH_PAID_PER_BYTE;
+    logEvent({
+      session: id,
+      type: "llm",
+      model,
+      cost_usd: cost,
+      first_ms: llmFirst,
+      ms: Math.round(performance.now() - t0),
+      speculative: speculative || undefined,
+    });
+    if (speaker.model)
+      logEvent({ session: id, type: "tts", model: speaker.model, paid_bytes: speaker.paidBytes, cost_usd: ttsCost });
+    emit({
+      type: "done",
+      llm_first_ms: llmFirst,
+      tts_first_ms: speaker.firstMs,
+      ms: Math.round(performance.now() - t0),
+      cost_usd: cost + ttsCost,
+      model,
+      tts_model: speaker.model,
+    });
+  } catch (e) {
+    const dropped = !!spec && !spec.committed;
+    if (cost)
+      logEvent({
+        session: id,
+        type: "llm",
+        model,
+        cost_usd: cost,
+        aborted: true,
+        speculative: speculative || undefined,
+      });
+    if (dropped) return;
+    if (ctrl.signal.aborted) {
+      /* interrupted: what was already said stays in the history, so the next turn knows */
+      if (partial) local.push({ role: "assistant", content: `${partial} [interrompido pelo usuário]` });
+      closeCalls(local);
+      s.messages.push(...local);
+      return;
+    }
+    emit({ type: "error", message: String((e as Error)?.message ?? e) });
+  } finally {
+    signal.removeEventListener("abort", abort);
+    if (s.turn === ctrl) s.turn = null;
+    if (s.spec === spec) s.spec = null;
+  }
+}