@crievo/omni 0.0.0-stage → 0.1.1

package/studio/web/lib/omni-cascade.ts

@@ -0,0 +1,1067 @@
+/* Omni in cascade mode in the browser (api/services/agent-cascade.ts has the why): the microphone
+   is heard here, and each turn's text, tools and audio stream back from /api/agent/cascade/:id/turn.
+   - stream: the microphone goes to xAI's STT WebSocket the whole time; its speech_final (Smart Turn)
+     ends the user's turn.
+   - chunks: this page finds each utterance (energy, then a pause) and posts it whole to
+     /api/agent/stt; nothing is sent while nobody speaks.
+   Same surface as OmniConversation (lib/omni-realtime), so useOmniConversation drives either one.
+   The reply plays through an <audio> element fed by a MediaStream, like Grok's, so the browser's echo
+   cancellation subtracts it from the microphone.
+   Waiting (wait_for_work, lib/omni-wait.ts): the STT socket closes and the silence timers stop; the
+   server session (its `messages`) stays. Speaking, typing or the awaited Claude stopping wakes it;
+   the first words heard while the STT reconnects are replayed into it. */
+
+import { api, enc, send } from "./api";
+import { openMic } from "./mic-device";
+import { greetingSamples } from "./omni-greeting-audio";
+import { currentPath, openScreen, runUiTool } from "./omni-ui";
+import type { NavigateArgs } from "../../shared/omni-screens";
+import {
+  historyMessages,
+  WAIT_OUTPUT,
+  WAIT_TOOL,
+  type WaitTarget,
+  waitHint,
+  waitTargetOf,
+  WakeDetector,
+  WorkWatch,
+  workDoneText,
+} from "./omni-wait";
+import type { AgentPendingAction, AgentResolvedAction } from "../types/api";
+import type { ConfirmResult, OmniCallbacks, OmniEntry, OmniStatus, OmniWait } from "./omni-realtime";
+
+export type CascadeMode = "stream" | "chunks";
+
+const RATE = 24000;
+const CHUNK_MS = 100;
+const WORKLET = `
+class Capture extends AudioWorkletProcessor {
+  constructor() { super(); this.buf = new Int16Array(2400); this.n = 0; }
+  process(inputs) {
+    const ch = inputs[0] && inputs[0][0];
+    if (ch) for (let i = 0; i < ch.length; i++) {
+      const s = Math.max(-1, Math.min(1, ch[i]));
+      this.buf[this.n++] = s < 0 ? s * 0x8000 : s * 0x7fff;
+      if (this.n === this.buf.length) { this.port.postMessage(this.buf.buffer, [this.buf.buffer]); this.buf = new Int16Array(2400); this.n = 0; }
+    }
+    return true;
+  }
+}
+registerProcessor("omni-cascade-capture", Capture);
+`;
+
+/* speech: RMS above SPEAK for 2 chunks (200 ms). While Omni talks, the microphone also hears what
+   the echo canceller leaves of its voice: `echo` follows that level, and speaking over Omni takes
+   ECHO_RATIO times it (a fixed 0.07 missed a real voice over Omni, 2026-10-17). */
+const SPEAK = 0.03;
+const ECHO_START = 0.02;
+const ECHO_RATIO = 2.5;
+const PREROLL = 3;
+/* chunks mode: an utterance waits this long after its text before it counts as the whole turn;
+   speaking again before that joins the two (a pause mid-sentence isn't the end of the turn).
+   The reply is already being prepared meanwhile, so this rarely delays the first audio. */
+const JOIN_MS = 900;
+
+const sec = (ms: number | null | undefined) => (ms == null ? "–" : `${(ms / 1000).toFixed(1).replace(".", ",")} s`);
+
+function toBase64(bytes: Uint8Array): string {
+  let s = "";
+  for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+  return btoa(s);
+}
+
+function fromBase64Pcm(b64: string): Float32Array<ArrayBuffer> {
+  const bin = atob(b64);
+  const n = bin.length >> 1;
+  const out = new Float32Array(n);
+  for (let i = 0; i < n; i++) {
+    const v = bin.charCodeAt(2 * i) | (bin.charCodeAt(2 * i + 1) << 8);
+    out[i] = (v >= 0x8000 ? v - 0x10000 : v) / 0x8000;
+  }
+  return out;
+}
+
+type TurnEvent =
+  | { type: "text"; delta: string }
+  | { type: "tool"; id: string; name: string; state: "running" | "ok" | "error"; ms?: number; detail?: string | null }
+  | { type: "pending"; action: AgentPendingAction }
+  | { type: "resolved"; resolved: AgentResolvedAction }
+  | { type: "open"; open: NavigateArgs }
+  | { type: "audio"; pcm: string }
+  | { type: "end_call" }
+  | { type: "ui_call"; id: string; name: string; args: unknown }
+  | {
+      type: "done";
+      llm_first_ms: number | null;
+      tts_first_ms: number | null;
+      ms: number;
+      cost_usd: number;
+      model: string;
+      tts_model: string | null;
+    }
+  | { type: "error"; message: string };
+
+type Speculation = {
+  ctrl: AbortController;
+  text: string;
+  committed: boolean;
+  queue: TurnEvent[];
+  handle: ((e: TurnEvent) => void) | null;
+  decide: ((go: boolean) => void) | null;
+  /* end of speech → the confirmed text, set on commit */
+  sttMs: number | null;
+};
+
+/* what the silence clips say (api/services/agent-cascade.ts CLIPS), for the echo check */
+const CLIP_TEXT = { nudge: "Ainda tá aí?", bye: "Vou encerrar por aqui. É só me chamar quando quiser." };
+
+const words = (t: string) =>
+  t
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .split(/\s+/)
+    .filter(Boolean);
+
+/* a transcript that is Omni's own words read back by the microphone: most of its words are in what
+   Omni just said. One or two words ("sim", "não", "pode") are never taken for an echo */
+function echoOf(text: string, said: string): boolean {
+  const heard = words(text);
+  if (heard.length < 3 || !said) return false;
+  const spoken = new Set(words(said));
+  return heard.filter(w => spoken.has(w)).length / heard.length >= 0.7;
+}
+
+/* the guess and the final text say the same thing (punctuation and case aside) */
+const same = (a: string, b: string) => {
+  const norm = (t: string) =>
+    t
+      .toLowerCase()
+      .replace(/[^\p{L}\p{N}\s]/gu, "")
+      .replace(/\s+/g, " ")
+      .trim();
+  return norm(a) === norm(b);
+};
+
+export class OmniCascade {
+  readonly analyser: AnalyserNode;
+  readonly micAnalyser: AnalyserNode;
+  sessionId: string | null = null;
+  private ctx: AudioContext;
+  private out: MediaStreamAudioDestinationNode;
+  private player: HTMLAudioElement;
+  private mic: MediaStream | null = null;
+  private worklet: AudioWorkletNode | null = null;
+  private stt: WebSocket | null = null;
+  private sttReady = false;
+  private sttRetries = 0;
+  private sttAudioS = 0;
+  private playing = new Set<AudioBufferSourceNode>();
+  private playhead = 0;
+  private entries: OmniEntry[] = [];
+  private status: OmniStatus = "idle";
+  private muted = false;
+  private closed = false;
+  private startedAt = 0;
+  private n = 0;
+  private turn: AbortController | null = null;
+  private ending = false;
+  private idleTimer: ReturnType<typeof setInterval> | null = null;
+  /* silence: since when nobody has said anything (Omni's own "Ainda tá aí?" doesn't restart it);
+     at nudgeMs Omni asks, at closeMs it says goodbye and hangs up (agent.cascade.idle) */
+  private silenceAt: number | null = null;
+  private nudgeMs = 6000;
+  private closeMs = 10_000;
+  private nudged = false;
+  private closing = false;
+  /* a silence clip is playing: the status it causes is not activity */
+  private clipActive = false;
+  private clips: Partial<Record<"nudge" | "bye", Float32Array<ArrayBuffer>>> = {};
+  /* right after Omni's voice stops, a final transcript is its echo, not the user */
+  private echoUntil = 0;
+  /* what Omni said last (its reply or a silence clip): an echo of it reads back as these words */
+  private said = "";
+  /* chunks mode: when the current utterance began */
+  private speechStartAt = 0;
+  private maxMs = 10 * 60_000;
+  private pauseMs = 700;
+  /* the local voice detector */
+  private loud = 0;
+  private inSpeech = false;
+  private quietMs = 0;
+  private lastVoiceAt = 0;
+  /* the user spoke over Omni (or while it thought): an STT final then is theirs, not an echo */
+  private heardUser = false;
+  private preroll: Int16Array[] = [];
+  private utterance: Int16Array[] = [];
+  /* the user bubble being filled (stream mode's interim text) */
+  private userEntry: string | null = null;
+  private finalWait: ReturnType<typeof setTimeout> | null = null;
+  /* stream mode: the text the STT already locked in this utterance (is_final chunks) */
+  private locked: string[] = [];
+  /* a reply started on that locked text before the STT's speech_final: its events wait here until
+     speech_final confirms the same text (commitSpec), or it is dropped (dropSpec) */
+  private spec: Speculation | null = null;
+  /* chunks mode: the residual echo heard while Omni talks */
+  private echo = ECHO_START;
+  /* chunks mode: the last utterance sent, and what is carried into the next one when the user resumed */
+  private lastUser: { entry: string; text: string } | null = null;
+  private carry: { entry: string; text: string } | null = null;
+  private joinTimer: ReturnType<typeof setTimeout> | null = null;
+  /* wait_for_work: the call sleeps once Omni finished saying so; asleep, `wait`; the time spent
+     asleep doesn't count toward max_session_min */
+  private waitFor: WaitTarget | null = null;
+  private wait: OmniWait | null = null;
+  private waitMaxMs = 20 * 60_000;
+  private waitedMs = 0;
+  private sleepTimer: ReturnType<typeof setInterval> | null = null;
+
+  /* `history`: an earlier conversation the user continues; it shows above and goes back into the
+     server session's messages */
+  constructor(
+    private cb: OmniCallbacks,
+    private mode: CascadeMode,
+    private history: OmniEntry[] = [],
+  ) {
+    this.entries = [...history];
+    this.ctx = new AudioContext({ sampleRate: RATE });
+    this.analyser = this.ctx.createAnalyser();
+    this.analyser.fftSize = 1024;
+    this.micAnalyser = this.ctx.createAnalyser();
+    this.micAnalyser.fftSize = 1024;
+    this.out = this.ctx.createMediaStreamDestination();
+    this.analyser.connect(this.out);
+    this.player = new Audio();
+    this.player.srcObject = this.out.stream;
+  }
+
+  async start() {
+    this.setStatus("connecting");
+    try {
+      await this.ctx.resume();
+      void this.player.play();
+      const mic = openMic();
+      const history = this.history.length ? historyMessages(this.history) : undefined;
+      const s = await send(api.agent.cascade.session.$post({ json: { mode: this.mode, history } }));
+      if (!s.ok || !("id" in s) || !s.id) {
+        void mic.then(m => m.getTracks().forEach(t => t.stop())).catch(() => {});
+        throw new Error(s.error ?? "não deu para abrir a sessão");
+      }
+      this.sessionId = s.id;
+      this.nudgeMs = (s.idle?.nudge_s ?? 6) * 1000;
+      this.closeMs = (s.idle?.close_s ?? 10) * 1000;
+      void this.loadClips();
+      this.maxMs = (s.config?.limits?.max_session_min ?? 10) * 60_000;
+      this.waitMaxMs = (s.config?.wait_max_min ?? 20) * 60_000;
+      this.pauseMs = s.chunk_pause_ms ?? 700;
+      this.mic = await mic;
+      if (this.mode === "stream" && s.stt) await this.openStt(s.stt.url, s.stt.token);
+      await this.startMic();
+      this.startedAt = performance.now();
+      this.push({
+        kind: "note",
+        id: "mode",
+        text:
+          this.mode === "stream"
+            ? "Cascata · STT em streaming → Haiku 5.5 → Fish"
+            : "Cascata · STT por trechos → Haiku 5.5 → Fish",
+      });
+      /* the greeting may still be playing */
+      this.setStatus(this.playing.size ? "speaking" : "listening");
+      this.idleTimer = setInterval(() => this.checkIdle(), 500);
+    } catch (e) {
+      this.fail(String((e as Error)?.message ?? e));
+    }
+  }
+
+  /* --- hearing ---------------------------------------------------------------------------------- */
+  private openStt(url: string, token: string) {
+    return new Promise<void>((resolve, reject) => {
+      this.sttReady = false;
+      const ws = new WebSocket(url, [`xai-client-secret.${token}`]);
+      this.stt = ws;
+      let opened = false;
+      ws.onerror = () => {
+        if (!opened) reject(new Error("a conexão com o STT da xAI falhou"));
+      };
+      ws.onclose = () => {
+        if (this.stt !== ws) return;
+        this.stt = null;
+        this.sttReady = false;
+        if (opened && !this.closed) void this.reconnectStt();
+      };
+      ws.onmessage = ev => {
+        let m: any;
+        try {
+          m = JSON.parse(String(ev.data));
+        } catch {
+          return;
+        }
+        if (m.type === "transcript.created") {
+          opened = true;
+          this.sttReady = true;
+          this.sttRetries = 0;
+          resolve();
+        } else if (m.type === "transcript.partial") this.onTranscript(m);
+        else if (m.type === "error") this.cb.onError(`STT: ${m.message ?? "erro"}`);
+      };
+    });
+  }
+
+  private async reconnectStt() {
+    if (++this.sttRetries > 5 || !this.sessionId) return this.fail("o STT da xAI caiu e não voltou");
+    const r = await send(api.agent.cascade[":id"].stt.$post({ param: { id: enc(this.sessionId) }, json: {} }));
+    if (this.closed) return;
+    if (!r.ok || !("url" in r)) return this.fail(r.error ?? "o STT da xAI caiu");
+    await this.openStt(r.url, r.token).catch(() => this.reconnectStt());
+  }
+
+  private onTranscript(m: { text?: string; is_final?: boolean; speech_final?: boolean }) {
+    const text = (m.text ?? "").trim();
+    if (!text) return;
+    /* while Omni speaks or a reply is on its way, only what the local detector heard as the user
+       counts (the rest is echo) */
+    const busy = this.playing.size > 0 || (this.turn != null && !this.guessing());
+    if (busy && !this.heardUser) return;
+    /* right after Omni stops, only its own words coming back are echo: a quick "sim" is the user */
+    if (!this.heardUser && performance.now() < this.echoUntil && echoOf(text, this.said)) return;
+    /* the bubble shows what is locked plus what is still being heard */
+    const shown = m.speech_final ? text : [...this.locked, text].join(" ");
+    if (!this.userEntry) {
+      this.userEntry = `u-${++this.n}`;
+      this.push({ kind: "user", id: this.userEntry, text: shown, pending: true });
+    } else this.setUser(this.userEntry, shown, !m.speech_final);
+    /* speech too quiet for the local detector: its last loud chunk is from before, so the turn starts now */
+    const now = performance.now();
+    const endAt = this.lastVoiceAt && now - this.lastVoiceAt < 4000 ? this.lastVoiceAt : now;
+    if (!m.speech_final) {
+      if (!m.is_final) return;
+      this.locked.push(text);
+      /* text locked and the user quiet: the reply starts on a guess (~0.2 s before speech_final) */
+      if (!this.inSpeech && !this.turn && !this.playing.size)
+        void this.runTurn(this.locked.join(" "), endAt, Math.round(now - endAt), false, true);
+      return;
+    }
+    const id = this.userEntry;
+    this.userEntry = null;
+    this.locked = [];
+    this.setUser(id, text, false);
+    if (this.finalWait) clearTimeout(this.finalWait);
+    if (this.guessing()) {
+      if (same(this.spec!.text, text)) return this.commitSpec(Math.round(now - endAt));
+      this.dropSpec();
+    }
+    void this.runTurn(text, endAt, Math.round(now - endAt));
+  }
+
+  /* a speculative reply is in flight and not confirmed yet */
+  private guessing() {
+    return !!this.spec && !this.spec.committed;
+  }
+
+  private commitSpec(sttMs: number) {
+    const spec = this.spec;
+    if (!spec || spec.committed || !this.sessionId) return;
+    spec.committed = true;
+    spec.sttMs = sttMs;
+    this.setStatus("thinking");
+    void send(api.agent.cascade[":id"].commit.$post({ param: { id: enc(this.sessionId) }, json: {} }));
+    const queued = spec.queue;
+    spec.queue = [];
+    for (const e of queued) spec.handle?.(e);
+    spec.decide?.(true);
+  }
+
+  /* the user went on talking (or said something else): the guess goes away unheard */
+  private dropSpec() {
+    const spec = this.spec;
+    if (!spec) return;
+    this.spec = null;
+    spec.ctrl.abort();
+    spec.decide?.(false);
+    if (this.turn === spec.ctrl) this.turn = null;
+  }
+
+  private async startMic() {
+    if (!this.mic) return;
+    const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" }));
+    await this.ctx.audioWorklet.addModule(url);
+    URL.revokeObjectURL(url);
+    const source = this.ctx.createMediaStreamSource(this.mic);
+    this.worklet = new AudioWorkletNode(this.ctx, "omni-cascade-capture");
+    this.worklet.port.onmessage = (e: MessageEvent<ArrayBuffer>) => this.onChunk(new Int16Array(e.data));
+    source.connect(this.worklet);
+    source.connect(this.micAnalyser);
+  }
+
+  private onChunk(pcm: Int16Array<ArrayBuffer>) {
+    if (this.closed) return;
+    if (this.wait) return this.onWaitChunk(pcm);
+    if (this.mode === "stream" && this.stt && this.sttReady) {
+      /* muted: silence keeps the STT session alive without hearing anything */
+      this.stt.send(this.muted ? new Int16Array(pcm.length) : pcm);
+      this.sttAudioS += CHUNK_MS / 1000;
+    }
+    if (this.muted) return;
+    let sum = 0;
+    for (let i = 0; i < pcm.length; i++) {
+      const v = pcm[i] / 0x8000;
+      sum += v * v;
+    }
+    const rms = Math.sqrt(sum / pcm.length);
+    const omniTalking = this.playing.size > 0;
+    if (!omniTalking) this.echo = ECHO_START;
+    const threshold = omniTalking ? Math.max(SPEAK, this.echo * ECHO_RATIO) : SPEAK;
+    /* what the microphone hears of Omni (anything under the bar), slowly averaged */
+    if (omniTalking && !this.inSpeech && rms <= threshold) this.echo = this.echo * 0.85 + rms * 0.15;
+    const need = 2;
+    if (rms > threshold) {
+      this.loud++;
+      this.quietMs = 0;
+      this.lastVoiceAt = performance.now();
+      if (!this.inSpeech && this.loud >= need) this.onSpeechStart();
+    } else {
+      this.loud = 0;
+      if (this.inSpeech && rms < SPEAK * 0.7) this.quietMs += CHUNK_MS;
+    }
+    if (this.inSpeech) {
+      if (this.mode === "chunks") this.utterance.push(pcm);
+      if (this.quietMs >= (this.mode === "chunks" ? this.pauseMs : 500)) this.onSpeechEnd();
+    } else if (this.mode === "chunks") {
+      this.preroll.push(pcm);
+      if (this.preroll.length > PREROLL) this.preroll.shift();
+    }
+  }
+
+  private onSpeechStart() {
+    this.inSpeech = true;
+    /* the user spoke before the call slept: it stays awake */
+    this.cancelSleep();
+    this.speechStartAt = performance.now() - CHUNK_MS * 2;
+    /* chunks: the user resumed before the last utterance counted as a whole turn: they go together */
+    if (this.mode === "chunks" && this.guessing() && this.lastUser) {
+      this.carry = this.lastUser;
+      this.lastUser = null;
+      if (this.joinTimer) clearTimeout(this.joinTimer);
+      this.dropSpec();
+    }
+    /* barge-in: Omni stops talking and the turn in flight is dropped */
+    if (this.playing.size || this.turn) {
+      this.heardUser = true;
+      this.interrupt();
+    }
+    if (this.mode === "chunks") {
+      this.utterance = [...this.preroll];
+      this.preroll = [];
+      this.userEntry = `u-${++this.n}`;
+      this.push({ kind: "user", id: this.userEntry, text: "", pending: true });
+    }
+    this.setStatus("user");
+  }
+
+  private onSpeechEnd() {
+    this.inSpeech = false;
+    this.quietMs = 0;
+    if (this.mode === "stream") {
+      /* the STT's speech_final ends the turn; until it comes, Omni waits (a noise never finalizes) */
+      if (this.status === "user") this.setStatus("thinking");
+      if (this.finalWait) clearTimeout(this.finalWait);
+      this.finalWait = setTimeout(() => {
+        if (this.guessing()) this.dropSpec();
+        if (!this.turn && !this.playing.size && this.status === "thinking") this.setStatus("listening");
+      }, 2500);
+      return;
+    }
+    const chunks = this.utterance;
+    this.utterance = [];
+    const id = this.userEntry;
+    this.userEntry = null;
+    /* no minimum length: speech already took 200 ms over the bar to start, and "sim" is short */
+    if (!id) {
+      this.dropUser(id);
+      if (this.carry) return this.sendCarry();
+      return this.setStatus("listening");
+    }
+    void this.transcribe(id, chunks, this.lastVoiceAt, this.speechStartAt);
+  }
+
+  private async transcribe(id: string, chunks: Int16Array[], endAt: number, startedAt = endAt) {
+    this.setStatus("thinking");
+    const length = chunks.reduce((n, c) => n + c.length, 0);
+    const all = new Int16Array(length);
+    let at = 0;
+    for (const c of chunks) {
+      all.set(c, at);
+      at += c.length;
+    }
+    const r = await send(
+      api.agent.stt.$post({
+        json: { session: this.sessionId ?? "-", audio: toBase64(new Uint8Array(all.buffer)) },
+      }),
+    );
+    if (this.closed) return;
+    const heard = r.ok && "text" in r ? String(r.text ?? "").trim() : "";
+    /* an utterance that began as Omni's voice stopped and reads as its own words: the echo, not the user */
+    const text = startedAt < this.echoUntil && echoOf(heard, this.said) ? "" : heard;
+    if (!text) {
+      this.dropUser(id);
+      if (this.carry && !this.inSpeech) return this.sendCarry();
+      if (!r.ok && r.error !== "áudio curto demais") this.cb.onError(r.error ?? "não entendi o áudio");
+      if (!this.turn && !this.playing.size) this.setStatus("listening");
+      return;
+    }
+    /* the user started talking again while this was transcribed: it goes with what comes next */
+    if (this.inSpeech) {
+      this.carry = this.joined(id, text);
+      return;
+    }
+    const turn = this.joined(id, text);
+    this.setUser(turn.entry, turn.text, false);
+    this.lastUser = turn;
+    const sttMs = Math.round(performance.now() - endAt);
+    const reply = this.runTurn(turn.text, endAt, sttMs, false, true);
+    if (this.joinTimer) clearTimeout(this.joinTimer);
+    this.joinTimer = setTimeout(() => {
+      this.lastUser = null;
+      this.commitSpec(sttMs);
+    }, JOIN_MS);
+    await reply;
+  }
+
+  /* what was carried had nothing after it (a noise): it is the turn on its own */
+  private sendCarry() {
+    const carry = this.carry;
+    this.carry = null;
+    if (!carry) return;
+    this.lastUser = null;
+    void this.runTurn(carry.text, this.lastVoiceAt || performance.now(), null);
+  }
+
+  /* this utterance, after what was carried from before: one bubble, one text */
+  private joined(id: string, text: string): { entry: string; text: string } {
+    const carry = this.carry;
+    this.carry = null;
+    if (!carry) return { entry: id, text };
+    this.dropUser(id);
+    const all = `${carry.text} ${text}`;
+    this.setUser(carry.entry, all, false);
+    return { entry: carry.entry, text: all };
+  }
+
+  /* --- a turn ------------------------------------------------------------------------------------ */
+  private async runTurn(text: string, endAt: number, sttMs: number | null, silent = false, speculative = false) {
+    if (!this.sessionId || this.closed) return;
+    this.interrupt();
+    this.heardUser = false;
+    const ctrl = new AbortController();
+    this.turn = ctrl;
+    const spec: Speculation | null = speculative
+      ? { ctrl, text, committed: false, queue: [], handle: null, decide: null, sttMs }
+      : null;
+    this.spec = spec;
+    if (!silent && !spec) this.setStatus("thinking");
+    const item = `a-${++this.n}`;
+    let latency: number | null = null;
+    const handle = (e: TurnEvent) => {
+      switch (e.type) {
+        case "text":
+          this.appendAssistant(item, e.delta);
+          break;
+        case "audio":
+          if (latency == null) {
+            latency = Math.round(performance.now() - endAt);
+            this.patchAssistant(item, { latency_ms: latency });
+          }
+          this.play(fromBase64Pcm(e.pcm));
+          this.setStatus("speaking");
+          break;
+        case "tool":
+          this.upsertTool(e);
+          if (!this.playing.size) this.setStatus("thinking");
+          break;
+        case "pending":
+          this.cb.onPending(e.action);
+          break;
+        case "resolved":
+          this.cb.onResolved?.(e.resolved);
+          break;
+        case "open":
+          void openScreen(e.open);
+          break;
+        case "end_call":
+          this.ending = true;
+          break;
+        case "ui_call":
+          void this.uiCall(e);
+          break;
+        case "done": {
+          const stt = spec ? spec.sttMs : sttMs;
+          const voice = e.tts_first_ms != null && e.llm_first_ms != null ? e.tts_first_ms - e.llm_first_ms : null;
+          const parts = [
+            stt != null ? `texto ${sec(stt)}` : null,
+            `LLM ${sec(e.llm_first_ms)}`,
+            `voz +${sec(voice)}`,
+            spec && this.mode === "stream" ? "antecipado" : null,
+            `US$ ${e.cost_usd.toFixed(4).replace(".", ",")}`,
+          ].filter(Boolean);
+          this.patchAssistant(item, { breakdown: parts.join(" · ") });
+          if (latency != null)
+            this.log({
+              type: "turn",
+              latency_ms: latency,
+              stt_ms: stt,
+              llm_first_ms: e.llm_first_ms,
+              tts_first_ms: e.tts_first_ms,
+              item,
+            });
+          break;
+        }
+        case "error":
+          this.cb.onError(e.message);
+          break;
+      }
+    };
+    if (spec) spec.handle = handle;
+    try {
+      const r = await fetch(`/api/agent/cascade/${enc(this.sessionId)}/turn`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ text, speculative: speculative || undefined, screen: currentPath() }),
+        signal: ctrl.signal,
+      });
+      if (!r.ok || !r.body) throw new Error(`o turno falhou (${r.status})`);
+      const reader = r.body.getReader();
+      const dec = new TextDecoder();
+      let buf = "";
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buf += dec.decode(value, { stream: true });
+        let nl = buf.indexOf("\n");
+        while (nl >= 0) {
+          const line = buf.slice(0, nl);
+          buf = buf.slice(nl + 1);
+          nl = buf.indexOf("\n");
+          if (!line.trim()) continue;
+          const e = JSON.parse(line) as TurnEvent;
+          if (ctrl.signal.aborted) return;
+          if (spec && !spec.committed) spec.queue.push(e);
+          else handle(e);
+        }
+      }
+      /* a guess whose reply finished before the STT confirmed it: wait for the verdict */
+      if (spec && !spec.committed && !(await new Promise<boolean>(r => (spec.decide = r)))) return;
+    } catch (e) {
+      if (ctrl.signal.aborted || this.closed) return;
+      this.cb.onError(String((e as Error)?.message ?? e));
+    } finally {
+      if (this.turn === ctrl) this.turn = null;
+      if (this.spec === spec) this.spec = null;
+    }
+    if (ctrl.signal.aborted || this.closed) return;
+    if (this.ending) return this.hangUpAfterSpeech();
+    if (this.waitFor) return this.sleepAfterSpeech();
+    if (!this.playing.size && this.status !== "user") this.setStatus("listening");
+  }
+
+  /* a UI tool the reply asked for (navigate): it runs here and the result goes back to the turn */
+  private async uiCall(e: Extract<TurnEvent, { type: "ui_call" }>) {
+    const id = `tool-${e.id}`;
+    this.upsertTool({ type: "tool", id, name: e.name, state: "running" });
+    const t0 = performance.now();
+    /* wait_for_work is this page's own: the call sleeps after the turn (runTurn) */
+    if (e.name === WAIT_TOOL) this.waitFor = waitTargetOf(e.args);
+    const output: Record<string, unknown> = e.name === WAIT_TOOL ? WAIT_OUTPUT : await runUiTool(e.name, e.args);
+    const error = typeof output.error === "string" ? output.error : null;
+    this.upsertTool({
+      type: "tool",
+      id,
+      name: e.name,
+      state: error ? "error" : "ok",
+      ms: Math.round(performance.now() - t0),
+      detail: error,
+    });
+    if (!this.sessionId) return;
+    void send(
+      api.agent.cascade[":id"]["ui-result"].$post({
+        param: { id: enc(this.sessionId) },
+        json: { call_id: e.id, output },
+      }),
+    );
+  }
+
+  /* the user spoke over Omni: the reply in flight stops where it was (a guess goes away unheard) */
+  private interrupt() {
+    if (this.guessing()) this.dropSpec();
+    this.turn?.abort();
+    this.turn = null;
+    this.spec = null;
+    this.stopPlayback();
+  }
+
+  setMuted(muted: boolean) {
+    this.muted = muted;
+  }
+
+  /* until the user first speaks in a call the panel opened by itself, its silence is the panel's to
+     handle (OmniWidget closes it quietly): no "Ainda tá aí?" and no idle hang-up here */
+  private idleHeld = false;
+  holdIdle(hold: boolean) {
+    this.idleHeld = hold;
+  }
+
+  /* the recorded greeting, as Omni's first words in a call the panel opened by itself: through the
+     call's output, so the user can talk over it like over any reply */
+  async greet(gender: string) {
+    const samples = await greetingSamples(this.ctx, gender);
+    if (!samples || this.closed || this.status === "user" || this.status === "error") return;
+    this.clipActive = true;
+    this.play(samples);
+    this.setStatus("speaking");
+  }
+
+  sendText(text: string) {
+    const t = text.trim();
+    if (!t) return;
+    this.push({ kind: "user", id: `u-${++this.n}`, text: t, pending: false });
+    if (this.wait) return void this.wake(t);
+    void this.runTurn(t, performance.now(), null);
+  }
+
+  async confirm(action: AgentPendingAction): Promise<ConfirmResult> {
+    const r = await send(api.agent.actions[":id"].confirm.$post({ param: { id: enc(action.id) }, json: {} }));
+    void this.runTurn(
+      r.ok
+        ? `[Tela] O usuário apertou Confirmar e deu certo: ${action.summary}. Resultado: ${(r.output ?? "").slice(-300)}.${waitHint(action.kind)}`
+        : `[Tela] O usuário apertou Confirmar, mas falhou: ${action.summary}. Erro: ${r.error}`,
+      performance.now(),
+      null,
+      true,
+    );
+    return r;
+  }
+
+  async cancel(action: AgentPendingAction) {
+    await send(api.agent.actions[":id"].cancel.$post({ param: { id: enc(action.id) }, json: {} }));
+    void this.runTurn(`[Tela] O usuário cancelou: ${action.summary}.`, performance.now(), null, true);
+  }
+
+  feedback(item: string, value: "up" | "down" | null) {
+    this.entries = this.entries.map(e => (e.kind === "assistant" && e.id === item ? { ...e, feedback: value } : e));
+    this.cb.onEntries(this.entries);
+    this.log({ type: "feedback", item, value: value ?? "none" });
+  }
+
+  private hangUpAfterSpeech() {
+    const wait = setInterval(() => {
+      if (this.playing.size) return;
+      clearInterval(wait);
+      this.push({ kind: "note", id: `end-${Date.now()}`, text: "O Omni encerrou a conversa." });
+      this.stop("agent");
+    }, 200);
+  }
+
+  private checkIdle() {
+    if (this.wait) {
+      if (!this.wait.waking && performance.now() - this.wait.since >= this.waitMaxMs) {
+        this.push({
+          kind: "note",
+          id: `wait-max-${Date.now()}`,
+          text: `Encerrei a conversa: ${Math.round(this.waitMaxMs / 60_000)} min esperando sem ninguém chamar.`,
+        });
+        this.stop("wait_max");
+      }
+      return;
+    }
+    if (this.startedAt && performance.now() - this.startedAt - this.waitedMs >= this.maxMs) {
+      this.push({
+        kind: "note",
+        id: `max-${Date.now()}`,
+        text: `Encerrei a conversa: chegou ao limite de ${Math.round(this.maxMs / 60_000)} min.`,
+      });
+      return this.stop("max");
+    }
+    if (this.idleHeld || this.status !== "listening" || this.silenceAt == null || this.closing) return;
+    const quiet = performance.now() - this.silenceAt;
+    if (quiet >= this.closeMs) {
+      this.closing = true;
+      this.playClip("bye");
+      const wait = setInterval(() => {
+        if (this.playing.size) return;
+        clearInterval(wait);
+        this.push({
+          kind: "note",
+          id: `idle-${Date.now()}`,
+          text: `Encerrei a conversa: ${Math.round(this.closeMs / 1000)} s sem ninguém falar.`,
+        });
+        this.stop("idle");
+      }, 200);
+      return;
+    }
+    if (!this.nudged && quiet >= this.nudgeMs) {
+      this.nudged = true;
+      this.playClip("nudge");
+    }
+  }
+
+  /* "Ainda tá aí?" and the goodbye, made once on the server in Omni's voice */
+  private async loadClips() {
+    for (const name of ["nudge", "bye"] as const) {
+      if (!this.sessionId) return;
+      try {
+        const r = await fetch(`/api/agent/cascade/${enc(this.sessionId)}/clip/${name}`);
+        if (!r.ok) continue;
+        const pcm = new Int16Array(await r.arrayBuffer());
+        const out = new Float32Array(pcm.length);
+        for (let i = 0; i < pcm.length; i++) out[i] = pcm[i] / 0x8000;
+        this.clips[name] = out;
+      } catch {
+        /* without the clip Omni just stays quiet (and still hangs up on time) */
+      }
+    }
+  }
+
+  private playClip(name: "nudge" | "bye") {
+    const clip = this.clips[name];
+    if (!clip) return;
+    this.said = CLIP_TEXT[name];
+    this.clipActive = true;
+    this.play(clip);
+    this.setStatus("speaking");
+  }
+
+  stop(reason = "user") {
+    if (this.closed) return;
+    this.closed = true;
+    if (this.idleTimer) clearInterval(this.idleTimer);
+    if (this.finalWait) clearTimeout(this.finalWait);
+    if (this.joinTimer) clearTimeout(this.joinTimer);
+    this.cancelSleep();
+    if (this.wait) {
+      this.waitedMs += performance.now() - this.wait.since;
+      this.wait.watch.stop();
+      this.wait = null;
+    }
+    if (this.startedAt)
+      this.log({
+        type: "end",
+        duration_s: Math.round((performance.now() - this.startedAt - this.waitedMs) / 100) / 10,
+        stt_audio_s: Math.round(this.sttAudioS * 10) / 10,
+        reason,
+      });
+    this.interrupt();
+    this.worklet?.disconnect();
+    this.mic?.getTracks().forEach(t => t.stop());
+    const stt = this.stt;
+    this.stt = null;
+    stt?.close(1000);
+    this.player.pause();
+    void this.ctx.close();
+    this.setStatus("idle");
+  }
+
+  /* --- waiting ------------------------------------------------------------------------------------ */
+  /* Omni said it will wait: the call sleeps once that sentence finished playing */
+  private sleepAfterSpeech() {
+    if (this.sleepTimer) return;
+    this.sleepTimer = setInterval(() => {
+      if (this.playing.size) return;
+      this.cancelSleep(false);
+      this.sleep();
+    }, 200);
+  }
+
+  /* `drop`: the user spoke first, so the wait is off */
+  private cancelSleep(drop = true) {
+    if (this.sleepTimer) clearInterval(this.sleepTimer);
+    this.sleepTimer = null;
+    if (drop) this.waitFor = null;
+  }
+
+  private sleep() {
+    const target = this.waitFor ?? { type: null, target: null };
+    this.waitFor = null;
+    if (this.closed || this.wait) return;
+    /* the STT stops hearing (stream mode bills every streamed second); the server session stays */
+    const stt = this.stt;
+    this.stt = null;
+    this.sttReady = false;
+    stt?.close(1000);
+    if (this.finalWait) clearTimeout(this.finalWait);
+    if (this.joinTimer) clearTimeout(this.joinTimer);
+    this.dropUser(this.userEntry);
+    this.userEntry = null;
+    this.inSpeech = false;
+    this.loud = 0;
+    this.quietMs = 0;
+    this.utterance = [];
+    this.preroll = [];
+    this.locked = [];
+    this.lastUser = null;
+    this.carry = null;
+    this.nudged = false;
+    const watch = new WorkWatch(target, s => void this.wake(workDoneText(s)));
+    this.wait = { since: performance.now(), detector: new WakeDetector(), watch, waking: false, held: [] };
+    this.push({ kind: "note", id: `wait-${Date.now()}`, text: "Esperando o estúdio. É só falar para me chamar." });
+    this.setStatus("waiting");
+    void watch.start();
+  }
+
+  private onWaitChunk(pcm: Int16Array<ArrayBuffer>) {
+    const w = this.wait;
+    if (!w) return;
+    if (w.waking) {
+      w.held.push(pcm);
+      return;
+    }
+    if (this.muted || !w.detector.feed(pcm)) return;
+    w.held = w.detector.take();
+    void this.wake(null);
+  }
+
+  /* `text`: what Omni hears first (the work's end, or a typed message); null: the user is speaking,
+     and what was heard while the STT came back is replayed into the detector and the STT */
+  private async wake(text: string | null) {
+    const w = this.wait;
+    if (!w || w.waking || this.closed) return;
+    w.waking = true;
+    w.watch.stop();
+    if (this.mode === "stream" && this.sessionId) {
+      this.setStatus("connecting");
+      const r = await send(api.agent.cascade[":id"].stt.$post({ param: { id: enc(this.sessionId) }, json: {} }));
+      if (this.closed) return;
+      if (!r.ok || !("url" in r)) return this.fail(r.error ?? "não deu para voltar a ouvir");
+      try {
+        await this.openStt(r.url, r.token);
+      } catch (e) {
+        return this.fail(String((e as Error)?.message ?? e));
+      }
+      if (this.closed) return;
+    }
+    this.waitedMs += performance.now() - w.since;
+    this.wait = null;
+    this.setStatus("listening");
+    if (text) return void this.runTurn(text, performance.now(), null, true);
+    for (const c of w.held) this.onChunk(c as Int16Array<ArrayBuffer>);
+  }
+
+  /* --- audio out ---------------------------------------------------------------------------------- */
+  private play(samples: Float32Array<ArrayBuffer>) {
+    if (!samples.length) return;
+    const buf = this.ctx.createBuffer(1, samples.length, RATE);
+    buf.copyToChannel(samples, 0);
+    const src = this.ctx.createBufferSource();
+    src.buffer = buf;
+    src.connect(this.analyser);
+    const at = Math.max(this.ctx.currentTime + 0.02, this.playhead);
+    src.start(at);
+    this.playhead = at + buf.duration;
+    this.playing.add(src);
+    src.onended = () => {
+      this.playing.delete(src);
+      if (this.playing.size) return;
+      this.echoUntil = performance.now() + 1200;
+      /* silent but still working (a search, a tool, the rest of the reply): "thinking" shimmers the avatar */
+      if (this.status === "speaking") this.setStatus(this.turn ? "thinking" : "listening");
+    };
+  }
+
+  private stopPlayback() {
+    for (const s of this.playing) {
+      s.onended = null;
+      try {
+        s.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+    this.playing.clear();
+    this.playhead = 0;
+  }
+
+  /* --- transcript ---------------------------------------------------------------------------------- */
+  private push(e: OmniEntry) {
+    if (this.entries.some(x => x.id === e.id && x.kind === e.kind)) return;
+    this.entries = [...this.entries, e];
+    this.cb.onEntries(this.entries);
+  }
+
+  private setUser(id: string, text: string, pending: boolean) {
+    this.entries = this.entries.map(x => (x.kind === "user" && x.id === id ? { ...x, text, pending } : x));
+    this.cb.onEntries(this.entries);
+  }
+
+  private dropUser(id: string | null) {
+    if (!id) return;
+    this.entries = this.entries.filter(x => !(x.kind === "user" && x.id === id));
+    this.cb.onEntries(this.entries);
+  }
+
+  private appendAssistant(id: string, delta: string) {
+    if (!this.entries.some(x => x.kind === "assistant" && x.id === id)) {
+      this.push({ kind: "assistant", id, text: "", latency_ms: null, feedback: null });
+      this.said = "";
+    }
+    this.said += delta;
+    this.entries = this.entries.map(x => (x.kind === "assistant" && x.id === id ? { ...x, text: x.text + delta } : x));
+    this.cb.onEntries(this.entries);
+  }
+
+  private patchAssistant(id: string, patch: { latency_ms?: number; breakdown?: string }) {
+    if (!this.entries.some(x => x.kind === "assistant" && x.id === id))
+      this.push({ kind: "assistant", id, text: "", latency_ms: null, feedback: null });
+    this.entries = this.entries.map(x => (x.kind === "assistant" && x.id === id ? { ...x, ...patch } : x));
+    this.cb.onEntries(this.entries);
+  }
+
+  private upsertTool(e: Extract<TurnEvent, { type: "tool" }>) {
+    const entry: OmniEntry = {
+      kind: "tool",
+      id: e.id,
+      name: e.name,
+      state: e.state,
+      detail: e.detail ?? null,
+      ms: e.ms ?? null,
+    };
+    if (this.entries.some(x => x.kind === "tool" && x.id === e.id))
+      this.entries = this.entries.map(x => (x.kind === "tool" && x.id === e.id ? entry : x));
+    else this.entries = [...this.entries, entry];
+    this.cb.onEntries(this.entries);
+  }
+
+  /* --- plumbing ------------------------------------------------------------------------------------ */
+  private setStatus(s: OmniStatus) {
+    if (this.status === s) return;
+    this.status = s;
+    if (s === "listening") {
+      if (!this.clipActive) this.silenceAt = performance.now();
+      this.clipActive = false;
+    } else if (!(s === "speaking" && this.clipActive)) this.silenceAt = null;
+    if (s === "user" || s === "thinking") this.nudged = false;
+    this.cb.onStatus(s);
+  }
+
+  private fail(message: string) {
+    this.cb.onError(message);
+    this.log({ type: "error", message });
+    this.stop("error");
+    this.setStatus("error");
+  }
+
+  private log(
+    body:
+      | {
+          type: "turn";
+          latency_ms: number;
+          stt_ms: number | null;
+          llm_first_ms: number | null;
+          tts_first_ms: number | null;
+          item?: string;
+        }
+      | { type: "feedback"; item: string; value: "up" | "down" | "none" }
+      | { type: "end"; duration_s: number; stt_audio_s?: number; reason?: string }
+      | { type: "error"; message: string },
+  ) {
+    if (!this.sessionId) return;
+    void send(api.agent.cascade[":id"].log.$post({ param: { id: enc(this.sessionId) }, json: body }));
+  }
+}