@schlessera/brain-ui-react 0.4.0

package/src/voice/asr-deepgram.ts

@@ -0,0 +1,325 @@
+// Deepgram streaming ASR client.
+// Captures mic audio with MediaRecorder (Opus) and pipes it to Deepgram's
+// streaming endpoint via WebSocket. The full connect URL (base params + domain
+// keyterms) and a short-lived token are minted server-side and injected here;
+// this client just opens the socket and emits partial + final transcript
+// events. Implements the @schlessera/brain-ui-sdk AsrClient surface.
+
+import type { AsrClient } from "@schlessera/brain-ui-sdk/client";
+import type { AsrEvent } from "@schlessera/brain-ui-sdk/protocol";
+
+export interface DeepgramClientOptions {
+  /** Full Deepgram streaming WS URL (base params + keyterms), from the server. */
+  url: string;
+  /** Short-lived Deepgram token, from the server. */
+  token: string;
+  onEvent: (evt: AsrEvent) => void;
+  onAudioLevel?: (level: number) => void;
+  onError?: (err: Error) => void;
+  onClose?: () => void;
+}
+
+export class DeepgramClient implements AsrClient {
+  private ws: WebSocket | null = null;
+  private recorder: MediaRecorder | null = null;
+  private stream: MediaStream | null = null;
+  private audioContext: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private levelRaf: number | null = null;
+  private opts: DeepgramClientOptions;
+  // Audio chunks captured before the Deepgram socket opens. The recorder starts
+  // as soon as the mic is granted; this holds the head of the utterance until
+  // it can be flushed on ws.onopen.
+  private pending: Blob[] = [];
+  private closed = false;
+  private drainResolve: (() => void) | null = null;
+  private wakeLock: WakeLockSentinel | null = null;
+  private wakeLockHandler: (() => void) | null = null;
+
+  constructor(opts: DeepgramClientOptions) {
+    this.opts = opts;
+  }
+
+  async start(): Promise<void> {
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        channelCount: 1,
+      },
+    });
+    // Stopped while the mic permission was resolving? Release the freshly
+    // granted stream now — otherwise it stays live with no reference left to
+    // stop it (cleanup() already ran before the stream existed).
+    if (this.closed) {
+      for (const track of stream.getTracks()) track.stop();
+      return;
+    }
+    this.stream = stream;
+
+    void this.acquireWakeLock();
+    this.startAudioLevelMeter(this.stream);
+
+    // Record from the moment the mic is live — chunks buffer in `pending` until
+    // the socket opens, so connect latency doesn't eat the first words.
+    this.startRecorder();
+
+    if (this.closed) return;
+
+    // Token is passed as a sec-websocket-protocol header per Deepgram spec.
+    this.ws = new WebSocket(this.opts.url, ["token", this.opts.token]);
+
+    this.ws.onopen = () => {
+      this.flushPending();
+    };
+
+    this.ws.onmessage = (evt) => {
+      try {
+        const data = JSON.parse(evt.data);
+        this.handleDeepgramMessage(data);
+      } catch (err) {
+        this.opts.onError?.(
+          err instanceof Error ? err : new Error("Bad Deepgram message")
+        );
+      }
+    };
+
+    this.ws.onerror = () => {
+      this.opts.onError?.(new Error("Deepgram socket error"));
+    };
+
+    this.ws.onclose = () => {
+      this.cleanup();
+      if (!this.closed) this.opts.onClose?.();
+    };
+  }
+
+  private startRecorder() {
+    if (!this.stream) return;
+    const mimeType = pickOpusMime();
+    this.recorder = new MediaRecorder(this.stream, {
+      mimeType,
+      audioBitsPerSecond: 32000,
+    });
+    this.recorder.ondataavailable = (evt) => {
+      if (evt.data.size === 0) return;
+      if (this.ws?.readyState === WebSocket.OPEN) {
+        this.ws.send(evt.data);
+      } else {
+        this.pending.push(evt.data);
+      }
+    };
+    // 250ms chunks — small enough to feel live, big enough to be Opus-frame-aligned
+    this.recorder.start(250);
+  }
+
+  private flushPending() {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
+    for (const chunk of this.pending) this.ws.send(chunk);
+    this.pending = [];
+  }
+
+  private handleDeepgramMessage(data: any) {
+    if (data.type === "Results") {
+      const alt = data.channel?.alternatives?.[0];
+      const transcript: string = alt?.transcript ?? "";
+      if (data.is_final) {
+        if (transcript) {
+          this.opts.onEvent({
+            type: "final",
+            text: transcript,
+            endsTurn: !!data.speech_final,
+          });
+        }
+        // If we are draining, this final settles the wait — but allow more
+        // arrivals up to the deadline. We resolve on Metadata or timeout.
+      } else if (transcript) {
+        this.opts.onEvent({ type: "partial", text: transcript });
+      }
+      return;
+    }
+    if (data.type === "UtteranceEnd") {
+      this.opts.onEvent({ type: "final", text: "", endsTurn: true });
+      return;
+    }
+    if (data.type === "Metadata") {
+      // Final summary — Deepgram sends this when the stream is closing.
+      this.drainResolve?.();
+      this.drainResolve = null;
+      return;
+    }
+  }
+
+  // Keep the screen on for the duration of capture so the phone doesn't sleep
+  // mid-sentence and clip the audio. The lock is released when the page becomes
+  // hidden, so re-acquire on visibility return.
+  private async acquireWakeLock(): Promise<void> {
+    if (typeof navigator === "undefined" || !("wakeLock" in navigator)) {
+      return;
+    }
+    try {
+      this.wakeLock = await navigator.wakeLock.request("screen");
+      this.wakeLock.addEventListener("release", () => {
+        this.wakeLock = null;
+      });
+      if (!this.wakeLockHandler) {
+        this.wakeLockHandler = () => {
+          if (
+            document.visibilityState === "visible" &&
+            !this.closed &&
+            !this.wakeLock
+          ) {
+            void this.acquireWakeLock();
+          }
+        };
+        document.addEventListener("visibilitychange", this.wakeLockHandler);
+      }
+    } catch (err) {
+      console.warn("[voice] Wake lock request failed:", err);
+    }
+  }
+
+  private releaseWakeLock(): void {
+    if (this.wakeLockHandler) {
+      document.removeEventListener("visibilitychange", this.wakeLockHandler);
+      this.wakeLockHandler = null;
+    }
+    if (this.wakeLock) {
+      this.wakeLock.release().catch(() => {});
+      this.wakeLock = null;
+    }
+  }
+
+  private startAudioLevelMeter(stream: MediaStream) {
+    if (!this.opts.onAudioLevel) return;
+    try {
+      const ctx = new AudioContext();
+      const src = ctx.createMediaStreamSource(stream);
+      const analyser = ctx.createAnalyser();
+      analyser.fftSize = 512;
+      src.connect(analyser);
+      this.audioContext = ctx;
+      this.analyser = analyser;
+      const buf = new Uint8Array(analyser.frequencyBinCount);
+      const tick = () => {
+        if (!this.analyser) return;
+        this.analyser.getByteTimeDomainData(buf);
+        let sum = 0;
+        for (let i = 0; i < buf.length; i++) {
+          const v = (buf[i] - 128) / 128;
+          sum += v * v;
+        }
+        const rms = Math.sqrt(sum / buf.length);
+        this.opts.onAudioLevel?.(Math.min(1, rms * 4));
+        this.levelRaf = requestAnimationFrame(tick);
+      };
+      tick();
+    } catch (err) {
+      console.warn("[voice] Audio level meter failed:", err);
+    }
+  }
+
+  stop() {
+    this.closed = true;
+    this.cleanup();
+  }
+
+  // Graceful stop: tells Deepgram to flush any buffered audio, waits for the
+  // remaining final results, then closes. Use this for "Done" — it preserves
+  // the tail of an utterance that hasn't been finalized yet.
+  async drainAndStop(maxWaitMs = 2000): Promise<void> {
+    this.closed = true;
+    if (this.recorder && this.recorder.state === "recording") {
+      try {
+        this.recorder.stop();
+      } catch {}
+    }
+    // If the socket is still connecting, give it a moment to open so the
+    // buffered head of the utterance can still be delivered and transcribed.
+    if (this.ws && this.ws.readyState === WebSocket.CONNECTING) {
+      await new Promise<void>((resolve) => {
+        const timer = setTimeout(resolve, maxWaitMs);
+        const settle = () => {
+          clearTimeout(timer);
+          resolve();
+        };
+        this.ws!.addEventListener("open", settle, { once: true });
+        this.ws!.addEventListener("close", settle, { once: true });
+      });
+    }
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      this.cleanup();
+      return;
+    }
+    this.flushPending();
+    await new Promise<void>((resolve) => {
+      let done = false;
+      const finish = () => {
+        if (done) return;
+        done = true;
+        this.drainResolve = null;
+        clearTimeout(timer);
+        resolve();
+      };
+      const timer = setTimeout(finish, maxWaitMs);
+      this.drainResolve = finish;
+      try {
+        // Finalize emits final results for buffered audio without closing.
+        this.ws!.send(JSON.stringify({ type: "Finalize" }));
+        // CloseStream then triggers a final Metadata message and closes.
+        this.ws!.send(JSON.stringify({ type: "CloseStream" }));
+      } catch {
+        finish();
+      }
+    });
+    this.cleanup();
+  }
+
+  private cleanup() {
+    this.releaseWakeLock();
+    if (this.levelRaf !== null) {
+      cancelAnimationFrame(this.levelRaf);
+      this.levelRaf = null;
+    }
+    if (this.recorder && this.recorder.state !== "inactive") {
+      try {
+        this.recorder.stop();
+      } catch {}
+    }
+    this.recorder = null;
+    this.pending = [];
+    if (this.stream) {
+      for (const track of this.stream.getTracks()) track.stop();
+      this.stream = null;
+    }
+    if (this.audioContext) {
+      this.audioContext.close().catch(() => {});
+      this.audioContext = null;
+    }
+    this.analyser = null;
+    if (this.ws && this.ws.readyState <= WebSocket.OPEN) {
+      try {
+        // Send Deepgram's CloseStream control message before closing
+        if (this.ws.readyState === WebSocket.OPEN) {
+          this.ws.send(JSON.stringify({ type: "CloseStream" }));
+        }
+        this.ws.close();
+      } catch {}
+    }
+    this.ws = null;
+  }
+}
+
+function pickOpusMime(): string {
+  const candidates = [
+    "audio/webm;codecs=opus",
+    "audio/ogg;codecs=opus",
+    "audio/webm",
+  ];
+  for (const c of candidates) {
+    if (typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(c)) {
+      return c;
+    }
+  }
+  return "audio/webm";
+}

package/src/voice/asr-webspeech.ts

@@ -0,0 +1,135 @@
+// Web Speech API ASR client — the `webspeech` provider. Recognition runs
+// entirely in the browser (no server session, no token), so this is the
+// zero-dependency / zero-cost STT path. Availability varies by browser
+// (Chrome/Edge yes, Firefox no), so callers must guard with
+// isWebSpeechAvailable() before selecting it.
+
+import type { AsrClient } from "@schlessera/brain-ui-sdk/client";
+import type { AsrEvent } from "@schlessera/brain-ui-sdk/protocol";
+
+// The Web Speech API isn't in the standard TS DOM lib; declare the minimal
+// surface we use.
+interface SpeechRecognitionAlternativeLike {
+  transcript: string;
+}
+interface SpeechRecognitionResultLike {
+  isFinal: boolean;
+  0: SpeechRecognitionAlternativeLike;
+}
+interface SpeechRecognitionEventLike {
+  resultIndex: number;
+  results: ArrayLike<SpeechRecognitionResultLike>;
+}
+interface SpeechRecognitionLike {
+  lang: string;
+  continuous: boolean;
+  interimResults: boolean;
+  start(): void;
+  stop(): void;
+  abort(): void;
+  onresult: ((event: SpeechRecognitionEventLike) => void) | null;
+  onerror: ((event: { error?: string }) => void) | null;
+  onend: (() => void) | null;
+}
+type SpeechRecognitionCtor = new () => SpeechRecognitionLike;
+
+function getRecognitionCtor(): SpeechRecognitionCtor | null {
+  if (typeof window === "undefined") return null;
+  const w = window as unknown as {
+    SpeechRecognition?: SpeechRecognitionCtor;
+    webkitSpeechRecognition?: SpeechRecognitionCtor;
+  };
+  return w.SpeechRecognition ?? w.webkitSpeechRecognition ?? null;
+}
+
+/** True when this browser supports the Web Speech API. */
+export function isWebSpeechAvailable(): boolean {
+  return getRecognitionCtor() !== null;
+}
+
+export interface WebSpeechClientOptions {
+  onEvent: (evt: AsrEvent) => void;
+  onError?: (err: Error) => void;
+  onEnd?: () => void;
+  /** BCP-47 language tag. Defaults to "en-US". */
+  lang?: string;
+}
+
+export class WebSpeechClient implements AsrClient {
+  private recognition: SpeechRecognitionLike | null = null;
+  private opts: WebSpeechClientOptions;
+  private closed = false;
+  private endResolvers: Array<() => void> = [];
+
+  constructor(opts: WebSpeechClientOptions) {
+    this.opts = opts;
+  }
+
+  async start(): Promise<void> {
+    const Ctor = getRecognitionCtor();
+    if (!Ctor) {
+      throw new Error("Web Speech API is not available in this browser.");
+    }
+    const recognition = new Ctor();
+    recognition.lang = this.opts.lang ?? "en-US";
+    recognition.continuous = true;
+    recognition.interimResults = true;
+
+    recognition.onresult = (event) => {
+      for (let i = event.resultIndex; i < event.results.length; i++) {
+        const result = event.results[i];
+        const transcript = result[0]?.transcript ?? "";
+        if (!transcript) continue;
+        if (result.isFinal) {
+          // No reliable end-of-utterance signal, so never auto-end the turn.
+          this.opts.onEvent({ type: "final", text: transcript, endsTurn: false });
+        } else {
+          this.opts.onEvent({ type: "partial", text: transcript });
+        }
+      }
+    };
+
+    recognition.onerror = (event) => {
+      if (this.closed) return;
+      this.opts.onError?.(new Error(`Speech recognition error: ${event.error ?? "unknown"}`));
+    };
+
+    recognition.onend = () => {
+      this.settleEnd();
+      if (!this.closed) this.opts.onEnd?.();
+    };
+
+    this.recognition = recognition;
+    recognition.start();
+  }
+
+  stop() {
+    this.closed = true;
+    try {
+      this.recognition?.abort();
+    } catch {}
+    this.recognition = null;
+    this.settleEnd();
+  }
+
+  async drainAndStop(): Promise<void> {
+    this.closed = true;
+    if (!this.recognition) return;
+    // stop() flushes the pending final result, then fires onend.
+    const done = new Promise<void>((resolve) => this.endResolvers.push(resolve));
+    try {
+      this.recognition.stop();
+    } catch {
+      this.settleEnd();
+    }
+    // Bound the wait so a browser that never fires onend can't hang the UI.
+    await Promise.race([done, new Promise<void>((r) => setTimeout(r, 1500))]);
+    this.recognition = null;
+  }
+
+  private settleEnd() {
+    const resolvers = this.endResolvers;
+    this.endResolvers = [];
+    for (const resolve of resolvers) resolve();
+  }
+}

package/src/voice/use-dictation.ts

@@ -0,0 +1,187 @@
+import { useCallback, useEffect, useRef } from "react";
+import { useVoiceStore } from "./voice-store.js";
+import {
+  createAsrClient,
+  speechUiHints,
+  type AsrClient,
+} from "@schlessera/brain-ui-sdk/client";
+import type { PronunciationOverride } from "@schlessera/brain-ui-sdk/protocol";
+import { api } from "../lib/api-client.js";
+import { registerAsrClients } from "./asr-clients.js";
+
+// Register the built-in ASR clients once at module load.
+registerAsrClients();
+
+/** Apply pronunciation overrides client-side to a finalized transcript. */
+function applyOverrides(
+  text: string,
+  overrides: PronunciationOverride[]
+): string {
+  if (overrides.length === 0) return text;
+  let out = text;
+  for (const { match, replacement } of overrides) {
+    if (!match) continue;
+    const escaped = match.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    out = out.replace(new RegExp(`\\b${escaped}\\b`, "gi"), replacement);
+  }
+  return out;
+}
+
+export function useDictation() {
+  const clientRef = useRef<AsrClient | null>(null);
+  const overridesRef = useRef<PronunciationOverride[]>([]);
+  // Bumped on every start() and on every stop/cancel/unmount. A start that
+  // discovers its generation has been superseded aborts instead of opening
+  // (or continuing to hold) the mic — this closes the window where a slow
+  // session fetch would open a MediaStream after the user already stopped.
+  const startGenRef = useRef(0);
+  // AbortController for the in-flight session fetch, so stop/cancel/unmount
+  // can actually cancel a pending connect rather than just ignoring its result.
+  const sessionAbortRef = useRef<AbortController | null>(null);
+  const setMode = useVoiceStore((s) => s.setMode);
+  const setConnecting = useVoiceStore((s) => s.setConnecting);
+  const setProviderId = useVoiceStore((s) => s.setProviderId);
+  const setPartial = useVoiceStore((s) => s.setPartial);
+  const appendFinal = useVoiceStore((s) => s.appendFinal);
+  const setError = useVoiceStore((s) => s.setError);
+  const resetCapture = useVoiceStore((s) => s.resetCapture);
+  const setReviewText = useVoiceStore((s) => s.setReviewText);
+
+  const start = useCallback(async () => {
+    const gen = ++startGenRef.current;
+    resetCapture();
+    setProviderId(null);
+    // Show the sheet immediately, but as "connecting" — the mic is still shut
+    // during the session fetch, so we must not present a live "Listening" state
+    // (or its pulsing dot) until capture is actually open.
+    setConnecting(true);
+    setMode("dictate");
+
+    const abort = new AbortController();
+    sessionAbortRef.current = abort;
+
+    try {
+      // Fetch the dictation session + pronunciation overrides. The AsrClient's
+      // start() opens the mic and buffers audio until its stream is ready, so
+      // words spoken during connect are not lost.
+      const [session, ov] = await Promise.all([
+        api.voiceSession(abort.signal),
+        api
+          .voiceOverrides(abort.signal)
+          .catch(() => ({ overrides: [] as PronunciationOverride[] })),
+      ]);
+      // Superseded by a Stop/Escape/backdrop/unmount while the session was in
+      // flight? Bail before opening the mic — a MediaStream created here would
+      // have no reference left to stop it.
+      if (startGenRef.current !== gen) return;
+      if (sessionAbortRef.current === abort) sessionAbortRef.current = null;
+      overridesRef.current = ov.overrides;
+      // Surface which provider actually served this session (e.g. a silent
+      // fallback to browser speech) — see the sheet's provider indicator.
+      setProviderId(session.providerId);
+
+      const hints = speechUiHints(session.capabilities);
+      const client = createAsrClient({
+        session,
+        onEvent: (evt) => {
+          if (evt.type === "partial") {
+            // Providers without interim results skip live partials.
+            if (hints.showPartials) {
+              setPartial(applyOverrides(evt.text, overridesRef.current));
+            }
+          } else if (evt.type === "final" && evt.text) {
+            appendFinal(applyOverrides(evt.text, overridesRef.current));
+          }
+        },
+        onError: (err) => setError(err.message),
+      });
+      // Publish the client before awaiting start() so a concurrent stop() can
+      // grab and close it (releasing its MediaStream) while the mic is opening.
+      clientRef.current = client;
+      await client.start();
+      // Stopped while the mic was opening? Close the client we just started —
+      // its start() honors the closed flag and releases the stream.
+      if (startGenRef.current !== gen) {
+        client.stop();
+        if (clientRef.current === client) clientRef.current = null;
+        return;
+      }
+      // Capture is live now — drop the connecting state so the sheet can show
+      // the honest "Listening" indicator.
+      setConnecting(false);
+    } catch (err) {
+      // A superseded start failing (e.g. the aborted session fetch) is
+      // expected — swallow it so it doesn't surface a spurious error.
+      if (startGenRef.current !== gen) return;
+      if (sessionAbortRef.current === abort) sessionAbortRef.current = null;
+      setError(err instanceof Error ? err.message : "Voice start failed");
+      setConnecting(false);
+      setMode("idle");
+    }
+  }, [
+    resetCapture,
+    setMode,
+    setConnecting,
+    setProviderId,
+    setPartial,
+    appendFinal,
+    setError,
+  ]);
+
+  const stop = useCallback(
+    async (commitToReview = true) => {
+      // Invalidate any in-flight start and cancel its session fetch, so a slow
+      // connect can't open the mic after the user has asked it to stop.
+      startGenRef.current++;
+      sessionAbortRef.current?.abort();
+      sessionAbortRef.current = null;
+
+      const client = clientRef.current;
+      clientRef.current = null;
+
+      const setDraining = useVoiceStore.getState().setDraining;
+      if (client) {
+        if (commitToReview) {
+          setDraining(true);
+          try {
+            await client.drainAndStop();
+          } finally {
+            setDraining(false);
+          }
+        } else {
+          client.stop();
+        }
+      }
+
+      const { finalText, partial, reviewText } = useVoiceStore.getState();
+      const merged = [finalText, partial].filter(Boolean).join(" ").trim();
+      setConnecting(false);
+      setMode("idle");
+      if (commitToReview && merged) {
+        // Append to any text already under review so a prompt can be built up
+        // across multiple record/edit rounds without losing earlier takes.
+        setReviewText(reviewText ? `${reviewText} ${merged}` : merged);
+      }
+      resetCapture();
+    },
+    [resetCapture, setMode, setConnecting, setReviewText]
+  );
+
+  const cancel = useCallback(() => {
+    void stop(false);
+  }, [stop]);
+
+  useEffect(() => {
+    return () => {
+      // Tear down on unmount: invalidate the in-flight start, abort its fetch,
+      // and stop the client so no MediaStream survives the component.
+      startGenRef.current++;
+      sessionAbortRef.current?.abort();
+      sessionAbortRef.current = null;
+      clientRef.current?.stop();
+      clientRef.current = null;
+    };
+  }, []);
+
+  return { start, stop, cancel };
+}

package/src/voice/voice-store.ts

@@ -0,0 +1,55 @@
+import { create } from "zustand";
+import type { VoiceMode } from "@schlessera/brain-ui-sdk/protocol";
+
+interface VoiceState {
+  mode: VoiceMode;
+  connecting: boolean;      // session fetch / mic open in progress (mic not yet live)
+  draining: boolean;        // post-Done flush in progress
+  partial: string;
+  finalText: string;        // accumulated final transcript for current capture
+  audioLevel: number;       // 0..1 RMS level for waveform
+  reviewText: string;       // text waiting in the review card (post-stop)
+  providerId: string | null; // active speech provider (from the session response)
+  error: string | null;
+
+  setMode: (mode: VoiceMode) => void;
+  setConnecting: (connecting: boolean) => void;
+  setDraining: (draining: boolean) => void;
+  setPartial: (text: string) => void;
+  appendFinal: (text: string) => void;
+  setAudioLevel: (level: number) => void;
+  setReviewText: (text: string) => void;
+  clearReview: () => void;
+  setProviderId: (providerId: string | null) => void;
+  setError: (err: string | null) => void;
+  resetCapture: () => void;
+}
+
+export const useVoiceStore = create<VoiceState>((set) => ({
+  mode: "idle",
+  connecting: false,
+  draining: false,
+  partial: "",
+  finalText: "",
+  audioLevel: 0,
+  reviewText: "",
+  providerId: null,
+  error: null,
+
+  setMode: (mode) => set({ mode }),
+  setConnecting: (connecting) => set({ connecting }),
+  setDraining: (draining) => set({ draining }),
+  setPartial: (partial) => set({ partial }),
+  appendFinal: (text) =>
+    set((s) => ({
+      finalText: (s.finalText ? s.finalText + " " : "") + text.trim(),
+      partial: "",
+    })),
+  setAudioLevel: (audioLevel) => set({ audioLevel }),
+  setReviewText: (reviewText) => set({ reviewText }),
+  clearReview: () => set({ reviewText: "" }),
+  setProviderId: (providerId) => set({ providerId }),
+  setError: (error) => set({ error }),
+  resetCapture: () =>
+    set({ partial: "", finalText: "", audioLevel: 0, error: null }),
+}));