@crievo/omni 0.0.0-stage → 0.1.1

package/studio/web/lib/omni-greeting-audio.ts

@@ -0,0 +1,24 @@
+/* The recorded greeting (GET /api/agent/voices/:gender/sample), decoded into the call's own audio, so a
+   call the panel opened by itself says it through the same output as Omni's replies: echo cancellation
+   takes it out of the microphone and the user can talk over it (barge-in), like any reply. Decoded once
+   per voice and rate. */
+
+const decoded = new Map<string, Promise<Float32Array<ArrayBuffer> | null>>();
+
+export function greetingSamples(ctx: AudioContext, gender: string): Promise<Float32Array<ArrayBuffer> | null> {
+  const key = `${gender}:${ctx.sampleRate}`;
+  let job = decoded.get(key);
+  if (!job) {
+    job = fetch(`/api/agent/voices/${encodeURIComponent(gender)}/sample`)
+      .then(r => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(String(r.status)))))
+      .then(buf => ctx.decodeAudioData(buf))
+      .then(audio => new Float32Array(audio.getChannelData(0)))
+      .catch(() => {
+        /* without it the call just starts listening; the next open tries again */
+        decoded.delete(key);
+        return null;
+      });
+    decoded.set(key, job);
+  }
+  return job;
+}

package/studio/web/lib/omni-hotword.ts

@@ -0,0 +1,231 @@
+/* "Oi, Omni" with the panel closed. While it is on (the switch in Omni's settings, per browser), the
+   microphone stays open and a local voice detector cuts the start of each utterance that follows a
+   pause (OnsetClipper: up to CLIP_MS). Only that start leaves the page: /api/agent/hotword transcribes
+   it and says whether it calls Omni (shared/omni-hotword.ts). One clip at a time; the rest of what the
+   user says never leaves.
+
+   Why not a model in the browser: Vosk's small Portuguese model, measured 2026-10-17 on the user's
+   cloned voice and real narrations, heard "omni" in "ônibus", "o homem" and "mini" about 4 times a
+   minute; the xAI transcription got every test phrase right with no false call. */
+
+import { api, errorText, send } from "./api";
+import { openMic } from "./mic-device";
+import { pcmToBase64 } from "./pcm-recorder";
+
+const RATE = 24000;
+const CHUNK_MS = 100;
+const WORKLET = `
+class Capture extends AudioWorkletProcessor {
+  constructor() { super(); this.buf = new Int16Array(2400); this.n = 0; }
+  process(inputs) {
+    const ch = inputs[0] && inputs[0][0];
+    if (ch) for (let i = 0; i < ch.length; i++) {
+      const s = Math.max(-1, Math.min(1, ch[i]));
+      this.buf[this.n++] = s < 0 ? s * 0x8000 : s * 0x7fff;
+      if (this.n === this.buf.length) { this.port.postMessage(this.buf.buffer, [this.buf.buffer]); this.buf = new Int16Array(2400); this.n = 0; }
+    }
+    return true;
+  }
+}
+registerProcessor("omni-hotword-capture", Capture);
+`;
+
+/* --- the start of an utterance -------------------------------------------------------------------- */
+/* a voice: two chunks over the bar (the room's noise raises it, like omni-wait's detector) */
+const SPEAK = 0.03;
+const NEED = 2;
+/* the quiet before it that makes it the start of something said, not the middle */
+const GAP_MS = 600;
+/* how much of it goes out: "oi, omni" takes about a second */
+const CLIP_MS = 2000;
+/* a pause this long ends it sooner */
+const END_MS = 500;
+/* chunks kept from before the voice crossed the bar, so the first syllable isn't lost */
+const KEEP = 4;
+
+export class OnsetClipper {
+  private kept: Int16Array[] = [];
+  private clip: Int16Array[] | null = null;
+  private loud = 0;
+  /* ms of quiet so far, and how much came before the current loud stretch */
+  private quiet = GAP_MS;
+  private quietBefore = 0;
+  private floor = 0.005;
+
+  /* the clip, once one is whole */
+  feed(pcm: Int16Array): Int16Array | null {
+    let sum = 0;
+    for (let i = 0; i < pcm.length; i++) {
+      const v = pcm[i] / 0x8000;
+      sum += v * v;
+    }
+    const rms = Math.sqrt(sum / pcm.length);
+    const loud = rms > Math.max(SPEAK, this.floor * 3);
+    if (!loud) this.floor = this.floor * 0.95 + rms * 0.05;
+    if (this.clip) {
+      this.clip.push(pcm);
+      this.quiet = loud ? 0 : this.quiet + CHUNK_MS;
+      if (this.clip.length * CHUNK_MS < CLIP_MS && this.quiet < END_MS) return null;
+      const whole = join(this.clip);
+      this.clip = null;
+      this.loud = 0;
+      return whole;
+    }
+    this.kept.push(pcm);
+    if (this.kept.length > KEEP) this.kept.shift();
+    if (loud) {
+      if (this.loud === 0) this.quietBefore = this.quiet;
+      this.quiet = 0;
+      if (++this.loud === NEED && this.quietBefore >= GAP_MS) {
+        this.clip = this.kept;
+        this.kept = [];
+      }
+    } else {
+      this.loud = 0;
+      this.quiet += CHUNK_MS;
+    }
+    return null;
+  }
+}
+
+function join(chunks: Int16Array[]): Int16Array {
+  const all = new Int16Array(chunks.reduce((n, c) => n + c.length, 0));
+  let at = 0;
+  for (const c of chunks) {
+    all.set(c, at);
+    at += c.length;
+  }
+  return all;
+}
+
+/* --- the listener ---------------------------------------------------------------------------------- */
+/* blocked: the browser holds the audio until the user clicks the page (autoplay policy, after a reload);
+   limit: the day's cap was reached, it stops until tomorrow */
+export type HotwordState = "starting" | "listening" | "blocked" | "limit" | "error";
+
+/* after a failed check (studio down, xAI out), the next clips wait this long */
+const BACKOFF_MS = 30_000;
+
+export class HotwordListener {
+  private ctx: AudioContext | null = null;
+  private stream: MediaStream | null = null;
+  private node: AudioWorkletNode | null = null;
+  private clipper = new OnsetClipper();
+  private checking = false;
+  private quietUntil = 0;
+  private stopped = false;
+  private unlock = () => void this.ctx?.resume();
+
+  constructor(private cb: { onCall: () => void; onState: (state: HotwordState, error?: string) => void }) {}
+
+  async start() {
+    this.cb.onState("starting");
+    try {
+      const stream = await openMic();
+      if (this.stopped) return stream.getTracks().forEach(t => t.stop());
+      this.stream = stream;
+      const ctx = new AudioContext({ sampleRate: RATE });
+      this.ctx = ctx;
+      const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" }));
+      await ctx.audioWorklet.addModule(url);
+      URL.revokeObjectURL(url);
+      if (this.stopped) return;
+      this.node = new AudioWorkletNode(ctx, "omni-hotword-capture");
+      this.node.port.onmessage = (e: MessageEvent<ArrayBuffer>) => this.onChunk(new Int16Array(e.data));
+      ctx.createMediaStreamSource(stream).connect(this.node);
+      ctx.onstatechange = () => this.syncState();
+      /* without a click since the page loaded, the context stays suspended: the next click starts it */
+      window.addEventListener("pointerdown", this.unlock, true);
+      window.addEventListener("keydown", this.unlock, true);
+      void ctx.resume();
+      setTimeout(() => this.syncState(), 300);
+    } catch (e) {
+      if (!this.stopped) this.cb.onState("error", micError(e));
+      this.stop();
+    }
+  }
+
+  stop() {
+    this.stopped = true;
+    window.removeEventListener("pointerdown", this.unlock, true);
+    window.removeEventListener("keydown", this.unlock, true);
+    this.node?.port.close();
+    this.node?.disconnect();
+    this.stream?.getTracks().forEach(t => t.stop());
+    void this.ctx?.close().catch(() => {});
+    this.ctx = null;
+    this.stream = null;
+    this.node = null;
+  }
+
+  private syncState() {
+    if (this.stopped || !this.ctx) return;
+    if (this.ctx.state === "running") {
+      window.removeEventListener("pointerdown", this.unlock, true);
+      window.removeEventListener("keydown", this.unlock, true);
+      this.cb.onState("listening");
+    } else this.cb.onState("blocked");
+  }
+
+  private onChunk(pcm: Int16Array) {
+    if (this.stopped) return;
+    const clip = this.clipper.feed(pcm);
+    /* one check at a time: a clip while one is out is dropped, not queued */
+    if (!clip || this.checking || performance.now() < this.quietUntil) return;
+    void this.check(clip);
+  }
+
+  private async check(clip: Int16Array) {
+    this.checking = true;
+    try {
+      const r = await send(api.agent.hotword.$post({ json: { audio: pcmToBase64(clip) } }));
+      if (this.stopped) return;
+      if (r.ok && "called" in r) {
+        this.cb.onState("listening");
+        if (r.called) this.cb.onCall();
+        return;
+      }
+      if ("limit" in r && r.limit) {
+        this.cb.onState("limit", r.error);
+        return this.stop();
+      }
+      this.quietUntil = performance.now() + BACKOFF_MS;
+      this.cb.onState("error", r.error ?? "não deu para ouvir agora");
+    } catch (e) {
+      if (this.stopped) return;
+      this.quietUntil = performance.now() + BACKOFF_MS;
+      this.cb.onState("error", errorText(e));
+    } finally {
+      this.checking = false;
+    }
+  }
+}
+
+function micError(e: unknown): string {
+  const name = (e as Error)?.name;
+  if (name === "NotAllowedError") return "o navegador não deixou usar o microfone";
+  if (name === "NotFoundError") return "nenhum microfone encontrado";
+  return errorText(e);
+}
+
+/* --- the switch, per browser ----------------------------------------------------------------------- */
+const STORAGE_KEY = "omni-hotword";
+
+export function loadHotwordOn(): boolean {
+  try {
+    return localStorage.getItem(STORAGE_KEY) === "on";
+  } catch {
+    return false;
+  }
+}
+
+export function saveHotwordOn(on: boolean) {
+  try {
+    if (on) localStorage.setItem(STORAGE_KEY, "on");
+    else localStorage.removeItem(STORAGE_KEY);
+  } catch {
+    /* private window: the switch lasts this page */
+  }
+}
+
+export const HOTWORD_STORAGE_KEY = STORAGE_KEY;

package/studio/web/lib/omni-panel.ts

@@ -0,0 +1,67 @@
+/* What the sidebar's Omni button and the widget share. The widget (mounted once in the app shell, so the
+   conversation survives navigation) owns the panel and the call; it publishes the face, whether the panel is
+   open, the ring status and the glow (OmniGlow) here, and registers the function that opens or closes the
+   panel and the one that reads Omni's voice level. The button reads the first and calls the others: minimized,
+   the call goes on in the button alone, with the same light as the panel. */
+
+import { useSyncExternalStore } from "react";
+import type { OmniGlowState } from "../components/Omni/OmniGlow";
+import type { OmniStatus } from "./omni-realtime";
+
+export type OmniPanelState = {
+  /* agent.enabled false: no button */
+  available: boolean;
+  face: string | null;
+  open: boolean;
+  ring: OmniStatus;
+  glow: OmniGlowState;
+};
+
+let state: OmniPanelState = { available: false, face: null, open: false, ring: "idle", glow: "off" };
+let toggler: (() => void) | null = null;
+let leveler: (() => number) | null = null;
+const listeners = new Set<() => void>();
+
+export function publishOmniPanel(next: OmniPanelState) {
+  if (
+    next.available === state.available &&
+    next.face === state.face &&
+    next.open === state.open &&
+    next.ring === state.ring &&
+    next.glow === state.glow
+  )
+    return;
+  state = next;
+  listeners.forEach(l => l());
+}
+
+export function registerOmniToggle(fn: (() => void) | null) {
+  toggler = fn;
+}
+
+export const toggleOmniPanel = () => toggler?.();
+
+/* Omni's voice, 0–1, for the glow; read every frame */
+export function registerOmniLevel(fn: (() => number) | null) {
+  leveler = fn;
+}
+
+export const omniLevel = () => leveler?.() ?? 0;
+
+export function useOmniPanel(): OmniPanelState {
+  return useSyncExternalStore(
+    l => {
+      listeners.add(l);
+      return () => listeners.delete(l);
+    },
+    () => state,
+  );
+}
+
+/* ask_in_chat (the UI tool): the widget opens the chat with the field focused; false when there is no call */
+export type ChatAsk = "link" | "text";
+let asker: ((what: ChatAsk) => boolean) | null = null;
+export function registerOmniAskChat(fn: ((what: ChatAsk) => boolean) | null) {
+  asker = fn;
+}
+export const askOmniChat = (what: ChatAsk) => asker?.(what) ?? false;