@crievo/omni 0.0.0-stage → 0.1.1

package/studio/web/lib/omni-realtime.ts

@@ -0,0 +1,811 @@
+/* The Omni conversation in the browser: one WebSocket to Grok Voice (xAI realtime), the microphone
+   in and the reply out, both PCM16 at 24 kHz. The studio API only mints the token and runs the
+   tools: Grok asks for a function, this forwards it to /api/agent/tools and returns the output.
+
+   The reply plays through an <audio> element fed by a MediaStream (not straight to the speakers):
+   that is what lets the browser's echo cancellation subtract it from the microphone, so Omni
+   doesn't hear itself and the user can still interrupt it.
+
+   Waiting (wait_for_work, lib/omni-wait.ts): after a request that keeps a studio Claude busy for
+   minutes, the socket closes (Grok bills every open second) and only the microphone stays, with a
+   local voice detector. Speaking, or the awaited Claude stopping, resumes the same session id with a
+   new socket that gets the conversation so far; the first words said while it reconnects are kept
+   and sent once it is open. */
+
+import { openMic } from "./mic-device";
+import { greetingSamples } from "./omni-greeting-audio";
+import { currentScreen, isUiTool, onScreenChange, openScreen, runUiTool } from "./omni-ui";
+import { api, enc, send } from "./api";
+import {
+  transcriptText,
+  WAIT_OUTPUT,
+  WAIT_TOOL,
+  type WaitTarget,
+  waitHint,
+  waitTargetOf,
+  WakeDetector,
+  WorkWatch,
+  workDoneText,
+} from "./omni-wait";
+import type { AgentPendingAction, AgentResolvedAction } from "../types/api";
+import type { NavigateArgs } from "../../shared/omni-screens";
+import { isFarewell } from "../../shared/omni-farewell";
+
+/* waiting: the call sleeps until the user speaks or the studio's work ends (no voice billed) */
+export type OmniStatus = "idle" | "connecting" | "listening" | "user" | "thinking" | "speaking" | "waiting" | "error";
+
+/* the call asleep: what wakes it and what was heard while it reconnects */
+export type OmniWait = {
+  since: number;
+  detector: WakeDetector;
+  watch: WorkWatch;
+  waking: boolean;
+  held: Int16Array[];
+};
+
+export type OmniEntry =
+  | { kind: "user"; id: string; text: string; pending: boolean }
+  | {
+      kind: "assistant";
+      id: string;
+      text: string;
+      latency_ms: number | null;
+      feedback: "up" | "down" | null;
+      /* cascade mode: where the latency went (text, LLM, voice) and the turn's cost */
+      breakdown?: string;
+    }
+  | {
+      kind: "tool";
+      id: string;
+      name: string;
+      state: "running" | "ok" | "error";
+      detail: string | null;
+      ms: number | null;
+    }
+  | { kind: "note"; id: string; text: string };
+
+/* the screen's Confirmar: open is the screen of what it created (a new content or reference) */
+export type ConfirmResult = { ok: boolean; output?: string; error?: string; open?: NavigateArgs | null };
+
+export type OmniCallbacks = {
+  onStatus: (s: OmniStatus) => void;
+  onEntries: (e: OmniEntry[]) => void;
+  onPending: (a: AgentPendingAction) => void;
+  /* a card decided by voice (confirm_action, cancel_action) */
+  onResolved?: (r: AgentResolvedAction) => void;
+  onError: (message: string) => void;
+  /* today's voice quota is used up (before or during the conversation): the widget goes to Claude mode */
+  onQuota?: () => void;
+};
+
+const RATE = 24000;
+
+/* the capture worklet: posts ~100 ms of PCM16 at a time */
+const WORKLET = `
+class Capture extends AudioWorkletProcessor {
+  constructor() { super(); this.buf = new Int16Array(2400); this.n = 0; }
+  process(inputs) {
+    const ch = inputs[0] && inputs[0][0];
+    if (ch) for (let i = 0; i < ch.length; i++) {
+      const s = Math.max(-1, Math.min(1, ch[i]));
+      this.buf[this.n++] = s < 0 ? s * 0x8000 : s * 0x7fff;
+      if (this.n === this.buf.length) { this.port.postMessage(this.buf.buffer, [this.buf.buffer]); this.buf = new Int16Array(2400); this.n = 0; }
+    }
+    return true;
+  }
+}
+registerProcessor("omni-capture", Capture);
+`;
+
+function safeJson(text: string): unknown {
+  try {
+    return JSON.parse(text || "{}");
+  } catch {
+    return {};
+  }
+}
+
+function toBase64(buf: ArrayBuffer): string {
+  const bytes = new Uint8Array(buf);
+  let s = "";
+  for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+  return btoa(s);
+}
+
+function fromBase64Pcm(b64: string): Float32Array<ArrayBuffer> {
+  const bin = atob(b64);
+  const n = bin.length >> 1;
+  const out = new Float32Array(n);
+  for (let i = 0; i < n; i++) {
+    const v = bin.charCodeAt(2 * i) | (bin.charCodeAt(2 * i + 1) << 8);
+    out[i] = (v >= 0x8000 ? v - 0x10000 : v) / 0x8000;
+  }
+  return out;
+}
+
+export class OmniConversation {
+  /* the reply audio (Omni's voice) */
+  readonly analyser: AnalyserNode;
+  /* the user's microphone: it only measures, it is not connected to the output (no echo) */
+  readonly micAnalyser: AnalyserNode;
+  sessionId: string | null = null;
+  private ctx: AudioContext;
+  private out: MediaStreamAudioDestinationNode;
+  private player: HTMLAudioElement;
+  private ws: WebSocket | null = null;
+  private mic: MediaStream | null = null;
+  private worklet: AudioWorkletNode | null = null;
+  private playing = new Set<AudioBufferSourceNode>();
+  private playhead = 0;
+  private entries: OmniEntry[] = [];
+  private status: OmniStatus = "idle";
+  private muted = false;
+  /* voice time: the open stretches already closed by a wait, plus the one open now (0: asleep) */
+  private activeMs = 0;
+  private segmentStart = 0;
+  /* wait_for_work was called: the call sleeps once Omni finishes saying so */
+  private waitFor: WaitTarget | null = null;
+  private wait: OmniWait | null = null;
+  private waitMaxMs = 20 * 60_000;
+  private sleepTimer: ReturnType<typeof setInterval> | null = null;
+  /* end of the user's speech: the turn latency runs from here to the reply's first audio */
+  private speechStoppedAt: number | null = null;
+  private calls = new Map<string, Promise<void>[]>();
+  private closed = false;
+  /* idle: when everything went quiet after Omni's last real reply; Grok's own "Ainda tá aí?"
+     (input_audio_buffer.timeout_triggered) doesn't restart it, only the user does */
+  private quietSince: number | null = null;
+  private nudging = false;
+  /* how many times the user spoke (or typed): sent with each tool call, confirm_action needs the user
+     to have spoken after the card appeared */
+  private heard = 0;
+  private hangupMs = 20_000;
+  /* agent.limits.max_session_min: no conversation runs past it */
+  private maxMs = 10 * 60_000;
+  /* what is left of today's voice quota when the conversation started */
+  private voiceLeftMs = Number.POSITIVE_INFINITY;
+  private idleTimer: ReturnType<typeof setInterval> | null = null;
+  /* Omni called end_call, or the user said goodbye: hang up as soon as the goodbye finishes playing */
+  private ending = false;
+  private hangingUp = false;
+  /* whether Grok is answering, and when it last finished and the user last stopped talking: a goodbye
+     whose transcript comes after Grok's reply hangs up right away */
+  private responding = false;
+  private lastDoneAt = 0;
+  private lastSpeechStop = 0;
+
+  /* `history`: an earlier conversation the user continues; it shows above and Grok gets it as context */
+  constructor(
+    private cb: OmniCallbacks,
+    private history: OmniEntry[] = [],
+  ) {
+    this.entries = [...history];
+    /* created in the click handler: Safari only plays audio from a context born in a gesture */
+    this.ctx = new AudioContext({ sampleRate: RATE });
+    this.analyser = this.ctx.createAnalyser();
+    this.analyser.fftSize = 1024;
+    this.micAnalyser = this.ctx.createAnalyser();
+    this.micAnalyser.fftSize = 1024;
+    this.out = this.ctx.createMediaStreamDestination();
+    this.analyser.connect(this.out);
+    this.player = new Audio();
+    this.player.srcObject = this.out.stream;
+  }
+
+  async start() {
+    this.setStatus("connecting");
+    try {
+      await this.ctx.resume();
+      void this.player.play();
+      const mic = openMic();
+      const s = await send(api.agent.session.$post({ json: {} }));
+      if (!s.ok && (s as { quota?: boolean }).quota) {
+        void mic.then(m => m.getTracks().forEach(t => t.stop())).catch(() => {});
+        this.closed = true;
+        this.setStatus("idle");
+        this.cb.onQuota?.();
+        return;
+      }
+      if (!s.ok || !("token" in s)) throw new Error(s.error ?? "não deu para abrir a sessão");
+      this.sessionId = s.id;
+      this.hangupMs = (s.config?.idle?.hangup_s ?? 20) * 1000;
+      this.maxMs = (s.config?.limits?.max_session_min ?? 10) * 60_000;
+      this.voiceLeftMs = (s.voice_remaining_s ?? 0) * 1000;
+      this.waitMaxMs = (s.config?.wait_max_min ?? 20) * 60_000;
+      this.mic = await mic;
+      await this.openSocket(s.url, s.token, s.session);
+      /* the studio's state for this session, as a system message (the console agent keeps its persona) */
+      this.sendEvent({
+        type: "conversation.item.create",
+        item: {
+          type: "message",
+          role: "system",
+          content: [{ type: "input_text", text: `${s.context}\n\n[Tela atual: ${currentScreen()}]` }],
+        },
+      });
+      if (this.history.length)
+        this.sendEvent({
+          type: "conversation.item.create",
+          item: {
+            type: "message",
+            role: "system",
+            content: [
+              {
+                type: "input_text",
+                text: `O usuário continuou uma conversa anterior com você. O estado do estúdio pode ter mudado desde então: confira com as ferramentas antes de afirmar algo. A conversa anterior:\n${transcriptText(this.history)}`,
+              },
+            ],
+          },
+        });
+      /* where the user goes next, so "esse vídeo" stays the one on screen */
+      this.unwatchScreen = onScreenChange(screen =>
+        this.sendEvent({
+          type: "conversation.item.create",
+          item: { type: "message", role: "system", content: [{ type: "input_text", text: `[Tela atual: ${screen}]` }] },
+        }),
+      );
+      await this.startMic();
+      this.segmentStart = performance.now();
+      /* the greeting may still be playing */
+      this.setStatus(this.playing.size ? "speaking" : "listening");
+      this.idleTimer = setInterval(() => this.checkIdle(), 500);
+    } catch (e) {
+      this.fail(String((e as Error)?.message ?? e));
+    }
+  }
+
+  private openSocket(url: string, token: string, session: Record<string, unknown>) {
+    return new Promise<void>((resolve, reject) => {
+      const ws = new WebSocket(url, [`xai-client-secret.${token}`]);
+      this.ws = ws;
+      ws.onopen = () => {
+        this.sendEvent({ type: "session.update", session });
+        resolve();
+      };
+      ws.onerror = () => reject(new Error("a conexão com o Grok falhou"));
+      ws.onclose = e => {
+        /* a socket closed on purpose (the wait) is no longer this.ws */
+        if (!this.closed && this.ws === ws) this.fail(`a conversa caiu (${e.code}${e.reason ? `: ${e.reason}` : ""})`);
+      };
+      ws.onmessage = ev => {
+        try {
+          this.onEvent(JSON.parse(String(ev.data)));
+        } catch {
+          /* a frame that isn't JSON: ignored */
+        }
+      };
+    });
+  }
+
+  private async startMic() {
+    if (!this.mic) return;
+    const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" }));
+    await this.ctx.audioWorklet.addModule(url);
+    URL.revokeObjectURL(url);
+    const source = this.ctx.createMediaStreamSource(this.mic);
+    this.worklet = new AudioWorkletNode(this.ctx, "omni-capture");
+    this.worklet.port.onmessage = (e: MessageEvent<ArrayBuffer>) => {
+      if (this.wait) return this.onWaitChunk(new Int16Array(e.data));
+      if (!this.muted) this.sendEvent({ type: "input_audio_buffer.append", audio: toBase64(e.data) });
+    };
+    source.connect(this.worklet);
+    source.connect(this.micAnalyser);
+  }
+
+  setMuted(muted: boolean) {
+    this.muted = muted;
+  }
+
+  /* until the user first speaks in a call the panel opened by itself, its silence is the panel's to
+     handle (OmniWidget closes it quietly): no "Ainda tá aí?" and no idle hang-up here */
+  private idleHeld = false;
+  holdIdle(hold: boolean) {
+    this.idleHeld = hold;
+  }
+
+  /* the recorded greeting, as Omni's first words in a call the panel opened by itself: through the
+     call's output, so the user can talk over it like over any reply */
+  async greet(gender: string) {
+    const samples = await greetingSamples(this.ctx, gender);
+    if (!samples || this.closed || this.status === "user" || this.status === "error") return;
+    this.play(samples);
+    this.setStatus("speaking");
+  }
+
+  /* a typed message instead of speech */
+  sendText(text: string) {
+    const t = text.trim();
+    if (!t) return;
+    this.heard++;
+    if (isFarewell(t)) {
+      /* typed: Grok's reply to it is still to come */
+      this.lastSpeechStop = performance.now();
+      this.farewell();
+    }
+    /* typing wakes a sleeping call, with this as the first thing it hears */
+    if (this.wait) {
+      this.push({ kind: "user", id: `text-${Date.now()}`, text: t, pending: false });
+      void this.wake(t);
+      return;
+    }
+    if (!this.ws) return;
+    this.nudging = false;
+    const id = `text-${Date.now()}`;
+    this.push({ kind: "user", id, text: t, pending: false });
+    this.sendEvent({
+      type: "conversation.item.create",
+      item: { type: "message", role: "user", content: [{ type: "input_text", text: t }] },
+    });
+    this.sendEvent({ type: "response.create" });
+    this.speechStoppedAt = performance.now();
+    this.setStatus("thinking");
+  }
+
+  async confirm(action: AgentPendingAction): Promise<ConfirmResult> {
+    const r = await send(api.agent.actions[":id"].confirm.$post({ param: { id: enc(action.id) }, json: {} }));
+    this.tellModel(
+      r.ok
+        ? `[Tela] O usuário apertou Confirmar e deu certo: ${action.summary}. Resultado: ${(r.output ?? "").slice(-300)}.${waitHint(action.kind)}`
+        : `[Tela] O usuário apertou Confirmar, mas falhou: ${action.summary}. Erro: ${r.error}`,
+    );
+    return r;
+  }
+
+  async cancel(action: AgentPendingAction) {
+    await send(api.agent.actions[":id"].cancel.$post({ param: { id: enc(action.id) }, json: {} }));
+    this.tellModel(`[Tela] O usuário cancelou: ${action.summary}.`);
+  }
+
+  /* something that happened on screen, for Omni to react to out loud */
+  private tellModel(text: string) {
+    if (!this.ws) return;
+    this.sendEvent({
+      type: "conversation.item.create",
+      item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
+    });
+    this.sendEvent({ type: "response.create" });
+  }
+
+  feedback(item: string, value: "up" | "down" | null) {
+    this.entries = this.entries.map(e => (e.kind === "assistant" && e.id === item ? { ...e, feedback: value } : e));
+    this.cb.onEntries(this.entries);
+    this.log({ type: "feedback", item, value: value ?? "none" });
+  }
+
+  /* the user said goodbye (shared/omni-farewell): the call ends once Grok's goodbye plays, whether or
+     not it calls end_call; with no reply in a few seconds, it ends anyway */
+  private farewell() {
+    this.ending = true;
+    if (!this.responding && this.lastDoneAt > this.lastSpeechStop) return this.hangUpAfterSpeech();
+    setTimeout(() => {
+      if (!this.closed && !this.responding) this.hangUpAfterSpeech();
+    }, 6000);
+  }
+
+  private hangUpAfterSpeech() {
+    if (this.hangingUp) return;
+    this.hangingUp = true;
+    const wait = setInterval(() => {
+      if (this.playing.size) return;
+      clearInterval(wait);
+      this.push({ kind: "note", id: `end-${Date.now()}`, text: "O Omni encerrou a conversa." });
+      this.stop("agent");
+    }, 200);
+  }
+
+  /* the voice time of this call: what was open, never the waits */
+  private elapsed() {
+    return this.activeMs + (this.segmentStart ? performance.now() - this.segmentStart : 0);
+  }
+
+  private checkIdle() {
+    if (this.wait) {
+      if (!this.wait.waking && performance.now() - this.wait.since >= this.waitMaxMs) {
+        this.push({
+          kind: "note",
+          id: `wait-max-${Date.now()}`,
+          text: `Encerrei a conversa: ${Math.round(this.waitMaxMs / 60_000)} min esperando sem ninguém chamar.`,
+        });
+        this.stop("wait_max");
+      }
+      return;
+    }
+    if (this.segmentStart && this.elapsed() >= this.voiceLeftMs) {
+      this.push({
+        kind: "note",
+        id: `quota-${Date.now()}`,
+        text: "Acabou a cota de voz de hoje: sigo por texto com o Claude.",
+      });
+      this.stop("quota");
+      this.cb.onQuota?.();
+      return;
+    }
+    if (this.segmentStart && this.elapsed() >= this.maxMs) {
+      this.push({
+        kind: "note",
+        id: `max-${Date.now()}`,
+        text: `Encerrei a conversa: chegou ao limite de ${Math.round(this.maxMs / 60_000)} min.`,
+      });
+      return this.stop("max");
+    }
+    if (this.idleHeld || this.status !== "listening" || this.quietSince == null) return;
+    if (performance.now() - this.quietSince < this.hangupMs) return;
+    this.push({
+      kind: "note",
+      id: `idle-${Date.now()}`,
+      text: `Encerrei a conversa: ${Math.round(this.hangupMs / 1000)} s sem ninguém falar.`,
+    });
+    this.stop("idle");
+  }
+
+  private unwatchScreen: (() => void) | null = null;
+
+  stop(reason = "user") {
+    if (this.closed) return;
+    this.closed = true;
+    if (this.idleTimer) clearInterval(this.idleTimer);
+    this.cancelSleep();
+    this.unwatchScreen?.();
+    this.wait?.watch.stop();
+    /* asleep, the open stretch already logged its end: this one closes the call (and its cards) */
+    if (this.segmentStart || this.wait)
+      this.log({
+        type: "end",
+        duration_s: this.segmentStart ? Math.round((performance.now() - this.segmentStart) / 100) / 10 : 0,
+        reason,
+      });
+    this.wait = null;
+    this.stopPlayback();
+    this.worklet?.disconnect();
+    this.mic?.getTracks().forEach(t => t.stop());
+    this.ws?.close(1000);
+    this.player.pause();
+    void this.ctx.close();
+    this.setStatus("idle");
+  }
+
+  /* --- events --------------------------------------------------------------------------------- */
+  private onEvent(e: any) {
+    switch (e.type) {
+      case "input_audio_buffer.speech_started":
+        /* barge-in: whatever is still queued stops; speaking before the call slept keeps it awake */
+        this.nudging = false;
+        this.heard++;
+        this.cancelSleep();
+        this.stopPlayback();
+        this.push({ kind: "user", id: e.item_id, text: "", pending: true });
+        this.setStatus("user");
+        break;
+      /* Grok re-engaging after the silence (idle_timeout_ms): its "Ainda tá aí?" */
+      case "input_audio_buffer.timeout_triggered":
+        this.nudging = true;
+        break;
+      case "input_audio_buffer.speech_stopped":
+        this.speechStoppedAt = performance.now();
+        this.lastSpeechStop = this.speechStoppedAt;
+        this.setStatus("thinking");
+        break;
+      case "conversation.item.input_audio_transcription.completed":
+        this.upsertUser(e.item_id, e.transcript ?? "");
+        if (isFarewell(e.transcript ?? "")) this.farewell();
+        break;
+      case "response.created":
+        this.responding = true;
+        break;
+      case "response.output_audio.delta":
+      case "response.audio.delta":
+        this.firstAudio(e.item_id);
+        this.play(fromBase64Pcm(e.delta));
+        break;
+      case "response.output_audio_transcript.delta":
+      case "response.audio_transcript.delta":
+        this.appendAssistant(e.item_id, e.delta ?? "");
+        break;
+      case "response.output_audio_transcript.done":
+      case "response.audio_transcript.done":
+        this.setAssistantText(e.item_id, e.transcript ?? "");
+        break;
+      case "response.function_call_arguments.done":
+        this.runCall(e.response_id, e.call_id, e.name, e.arguments ?? "{}");
+        break;
+      case "response.done":
+        this.responding = false;
+        this.lastDoneAt = performance.now();
+        void this.afterResponse(e.response?.id);
+        break;
+      case "error":
+        this.cb.onError(e.error?.message ?? e.message ?? "erro do Grok");
+        this.log({ type: "error", message: String(e.error?.message ?? e.message ?? "erro") });
+        break;
+    }
+  }
+
+  private firstAudio(item: string) {
+    const a = this.entries.find(x => x.kind === "assistant" && x.id === item);
+    if (!a) this.push({ kind: "assistant", id: item, text: "", latency_ms: null, feedback: null });
+    if (this.speechStoppedAt != null) {
+      const latency = Math.round(performance.now() - this.speechStoppedAt);
+      this.speechStoppedAt = null;
+      this.entries = this.entries.map(x =>
+        x.kind === "assistant" && x.id === item ? { ...x, latency_ms: latency } : x,
+      );
+      this.cb.onEntries(this.entries);
+      this.log({ type: "turn", latency_ms: latency, item });
+    }
+    this.setStatus("speaking");
+  }
+
+  private runCall(responseId: string, callId: string, name: string, args: string) {
+    if (name === "end_call") {
+      this.ending = true;
+      void send(
+        api.agent.tools.$post({ json: { session: this.sessionId ?? "-", call_id: callId, name, arguments: args } }),
+      );
+      return;
+    }
+    const id = `tool-${callId}`;
+    if (name === WAIT_TOOL) {
+      /* the browser's own: the call sleeps after Omni says it will wait (afterResponse) */
+      this.waitFor = waitTargetOf(safeJson(args));
+      this.push({ kind: "tool", id, name, state: "ok", detail: null, ms: 0 });
+      const list = this.calls.get(responseId) ?? [];
+      list.push(
+        Promise.resolve().then(() =>
+          this.sendEvent({
+            type: "conversation.item.create",
+            item: { type: "function_call_output", call_id: callId, output: JSON.stringify(WAIT_OUTPUT) },
+          }),
+        ),
+      );
+      this.calls.set(responseId, list);
+      return;
+    }
+    this.push({ kind: "tool", id, name, state: "running", detail: null, ms: null });
+    this.setStatus("thinking");
+    const job = (async () => {
+      /* a UI tool (navigate) runs here, on the screen; the rest in the studio API */
+      const t0 = performance.now();
+      const r: { ok: boolean; output?: unknown; error?: string; ms?: number } = isUiTool(name)
+        ? { ok: true, output: await runUiTool(name, safeJson(args)), ms: Math.round(performance.now() - t0) }
+        : await send(
+            api.agent.tools.$post({
+              json: { session: this.sessionId ?? "-", call_id: callId, name, arguments: args, heard: this.heard },
+            }),
+          );
+      const output: Record<string, unknown> =
+        r.ok && "output" in r ? (r.output as Record<string, unknown>) : { error: r.error ?? "falhou" };
+      const error = typeof output.error === "string" ? output.error : null;
+      this.entries = this.entries.map(x =>
+        x.kind === "tool" && x.id === id
+          ? { ...x, state: error ? "error" : "ok", detail: error, ms: "ms" in r ? Number(r.ms) : null }
+          : x,
+      );
+      this.cb.onEntries(this.entries);
+      if (output.pending_action) this.cb.onPending(output.pending_action as AgentPendingAction);
+      if (output.resolved) this.cb.onResolved?.(output.resolved as AgentResolvedAction);
+      /* a tool that changed something by itself (set_omnichat): the screen that shows it */
+      if (output.open) void openScreen(output.open as NavigateArgs);
+      this.sendEvent({
+        type: "conversation.item.create",
+        item: { type: "function_call_output", call_id: callId, output: JSON.stringify(output) },
+      });
+    })();
+    const list = this.calls.get(responseId) ?? [];
+    list.push(job);
+    this.calls.set(responseId, list);
+  }
+
+  /* every output of a response's calls goes back before the next response.create */
+  private async afterResponse(responseId: string | undefined) {
+    if (this.ending) return this.hangUpAfterSpeech();
+    const jobs = responseId ? this.calls.get(responseId) : undefined;
+    if (!jobs?.length) {
+      if (this.waitFor) return this.sleepAfterSpeech();
+      if (this.status !== "speaking") this.setStatus("listening");
+      return;
+    }
+    this.calls.delete(responseId!);
+    await Promise.all(jobs);
+    this.speechStoppedAt = performance.now();
+    this.sendEvent({ type: "response.create" });
+  }
+
+  /* --- waiting -------------------------------------------------------------------------------- */
+  /* Omni said it will wait: the call sleeps once that sentence finished playing */
+  private sleepAfterSpeech() {
+    if (this.sleepTimer) return;
+    this.sleepTimer = setInterval(() => {
+      if (this.playing.size) return;
+      this.cancelSleep(false);
+      this.sleep();
+    }, 200);
+  }
+
+  /* `drop`: the user spoke first, so the wait is off */
+  private cancelSleep(drop = true) {
+    if (this.sleepTimer) clearInterval(this.sleepTimer);
+    this.sleepTimer = null;
+    if (drop) this.waitFor = null;
+  }
+
+  private sleep() {
+    const target = this.waitFor ?? { type: null, target: null };
+    this.waitFor = null;
+    if (this.closed || this.wait) return;
+    const open = this.segmentStart ? performance.now() - this.segmentStart : 0;
+    this.activeMs += open;
+    this.segmentStart = 0;
+    /* this stretch's voice time ends here; the cards stay (reason "wait") */
+    this.log({ type: "end", duration_s: Math.round(open / 100) / 10, reason: "wait" });
+    const ws = this.ws;
+    this.ws = null;
+    ws?.close(1000);
+    this.calls.clear();
+    this.speechStoppedAt = null;
+    const watch = new WorkWatch(target, s => void this.wake(workDoneText(s)));
+    this.wait = { since: performance.now(), detector: new WakeDetector(), watch, waking: false, held: [] };
+    this.push({ kind: "note", id: `wait-${Date.now()}`, text: "Esperando o estúdio. É só falar para me chamar." });
+    this.setStatus("waiting");
+    void watch.start();
+  }
+
+  private onWaitChunk(pcm: Int16Array) {
+    const w = this.wait;
+    if (!w) return;
+    if (w.waking) {
+      w.held.push(pcm);
+      return;
+    }
+    if (this.muted || !w.detector.feed(pcm)) return;
+    w.held = w.detector.take();
+    void this.wake(null);
+  }
+
+  /* `text`: what Omni hears first (the work's end, or a typed message); null: the user is speaking,
+     and what was heard meanwhile goes in as audio */
+  private async wake(text: string | null) {
+    const w = this.wait;
+    if (!w || w.waking || this.closed) return;
+    w.waking = true;
+    w.watch.stop();
+    this.setStatus("connecting");
+    const s = await send(api.agent.session.$post({ json: { resume: this.sessionId ?? undefined } }));
+    if (this.closed) return;
+    if (!s.ok && (s as { quota?: boolean }).quota) {
+      this.push({
+        kind: "note",
+        id: `quota-${Date.now()}`,
+        text: "Acabou a cota de voz de hoje: sigo por texto com o Claude.",
+      });
+      this.stop("quota");
+      this.cb.onQuota?.();
+      return;
+    }
+    if (!s.ok || !("token" in s)) return this.fail(s.error ?? "não deu para voltar à conversa");
+    try {
+      await this.openSocket(s.url, s.token, s.session);
+    } catch (e) {
+      return this.fail(String((e as Error)?.message ?? e));
+    }
+    if (this.closed) return;
+    /* the quota counts what was already spoken in this call */
+    this.voiceLeftMs = this.activeMs + (s.voice_remaining_s ?? 0) * 1000;
+    this.sendEvent({
+      type: "conversation.item.create",
+      item: {
+        type: "message",
+        role: "system",
+        content: [
+          {
+            type: "input_text",
+            text: `${s.context}\n\n[Tela atual: ${currentScreen()}]\n\nVocê estava esperando o trabalho do estúdio com a conversa em pausa e voltou agora. Não cumprimente de novo. A conversa até aqui:\n${transcriptText(this.entries)}`,
+          },
+        ],
+      },
+    });
+    this.wait = null;
+    this.segmentStart = performance.now();
+    if (text) {
+      this.tellModel(text);
+      this.speechStoppedAt = performance.now();
+      this.setStatus("thinking");
+      return;
+    }
+    for (const c of w.held)
+      this.sendEvent({ type: "input_audio_buffer.append", audio: toBase64(c.buffer as ArrayBuffer) });
+    this.setStatus("listening");
+  }
+
+  /* --- audio out ------------------------------------------------------------------------------ */
+  private play(samples: Float32Array<ArrayBuffer>) {
+    if (!samples.length) return;
+    const buf = this.ctx.createBuffer(1, samples.length, RATE);
+    buf.copyToChannel(samples, 0);
+    const src = this.ctx.createBufferSource();
+    src.buffer = buf;
+    src.connect(this.analyser);
+    const at = Math.max(this.ctx.currentTime + 0.02, this.playhead);
+    src.start(at);
+    this.playhead = at + buf.duration;
+    this.playing.add(src);
+    src.onended = () => {
+      this.playing.delete(src);
+      if (!this.playing.size && this.status === "speaking") this.setStatus("listening");
+    };
+  }
+
+  private stopPlayback() {
+    for (const s of this.playing) {
+      s.onended = null;
+      try {
+        s.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+    this.playing.clear();
+    this.playhead = 0;
+  }
+
+  /* --- transcript ----------------------------------------------------------------------------- */
+  private push(e: OmniEntry) {
+    if (this.entries.some(x => x.id === e.id && x.kind === e.kind)) return;
+    this.entries = [...this.entries, e];
+    this.cb.onEntries(this.entries);
+  }
+
+  private upsertUser(id: string, text: string) {
+    if (!this.entries.some(x => x.kind === "user" && x.id === id))
+      this.push({ kind: "user", id, text, pending: false });
+    else this.entries = this.entries.map(x => (x.kind === "user" && x.id === id ? { ...x, text, pending: false } : x));
+    /* speech the server didn't keep (a noise): the empty bubble goes away */
+    this.entries = this.entries.filter(x => !(x.kind === "user" && !x.pending && !x.text.trim()));
+    this.cb.onEntries(this.entries);
+  }
+
+  private appendAssistant(id: string, delta: string) {
+    if (!this.entries.some(x => x.kind === "assistant" && x.id === id))
+      this.push({ kind: "assistant", id, text: "", latency_ms: null, feedback: null });
+    this.entries = this.entries.map(x => (x.kind === "assistant" && x.id === id ? { ...x, text: x.text + delta } : x));
+    this.cb.onEntries(this.entries);
+  }
+
+  private setAssistantText(id: string, text: string) {
+    if (!text) return;
+    this.entries = this.entries.map(x => (x.kind === "assistant" && x.id === id ? { ...x, text } : x));
+    this.cb.onEntries(this.entries);
+  }
+
+  /* --- plumbing ------------------------------------------------------------------------------- */
+  private sendEvent(e: Record<string, unknown>) {
+    if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(e));
+  }
+
+  private setStatus(s: OmniStatus) {
+    if (this.status === s) return;
+    this.status = s;
+    if (s === "listening") {
+      if (this.nudging) this.nudging = false;
+      else this.quietSince = performance.now();
+    } else if (!this.nudging) this.quietSince = null;
+    this.cb.onStatus(s);
+  }
+
+  private fail(message: string) {
+    this.cb.onError(message);
+    this.log({ type: "error", message });
+    this.stop("error");
+    this.setStatus("error");
+  }
+
+  private log(
+    body:
+      | { type: "turn"; latency_ms: number; item?: string }
+      | { type: "feedback"; item: string; value: "up" | "down" | "none" }
+      | { type: "end"; duration_s: number; reason?: string }
+      | { type: "error"; message: string },
+  ) {
+    if (!this.sessionId) return;
+    void send(api.agent.sessions[":id"].log.$post({ param: { id: enc(this.sessionId) }, json: body }));
+  }
+}