@unoverse-platform/studio 0.1.0

package/vendor/sdk/lib/realtime/useAudioCapture.ts

@@ -0,0 +1,237 @@
+/**
+ * useAudioCapture — continuous microphone capture for full-duplex voice.
+ *
+ * Streams raw mic audio continuously while a call is active. The remote provider
+ * (Grok / Nova) handles VAD server-side, so we don't gate locally. Audio is
+ * downsampled to 16 kHz mono Int16 PCM and emitted in ~80 ms chunks via
+ * `onAudioData`.
+ *
+ * Ported verbatim from gravity-client/src/realtime/useAudioCapture.ts (the proven,
+ * live implementation) — the Unoverse `voice` service reuses it unchanged.
+ */
+
+import { useRef, useCallback, useEffect, useState } from "react";
+import { float32ToInt16, downsampleFloat32 } from "./audioUtils";
+
+const TARGET_SAMPLE_RATE = 16000;
+// ~80 ms of audio at 16 kHz: 1280 samples
+const CHUNK_SAMPLES = 1280;
+
+export interface UseAudioCaptureOptions {
+  /** Callback when an audio chunk is captured (PCM Int16 16 kHz mono ArrayBuffer) */
+  onAudioData?: (audioData: ArrayBuffer) => void;
+  /** Whether to mute capture. Defaults to false — full-duplex barge-in. */
+  isMuted?: boolean;
+}
+
+export interface UseAudioCaptureReturn {
+  startCapture: () => Promise<{ success: boolean; reason?: string }>;
+  stopCapture: () => Promise<{ success: boolean; reason?: string }>;
+  toggleCapture: () => Promise<{ success: boolean; reason?: string }>;
+  /** Manually mute / unmute outgoing mic frames (keeps the stream open) */
+  setMuted: (muted: boolean) => void;
+  isCapturing: boolean;
+  /** Server-driven (Grok / Nova VAD); updated externally via onAudioState. */
+  isSpeaking: boolean;
+  isLoading: boolean;
+  error: string | null;
+}
+
+export function useAudioCapture(options: UseAudioCaptureOptions = {}): UseAudioCaptureReturn {
+  const { onAudioData, isMuted = false } = options;
+
+  const [isCapturing, setIsCapturing] = useState(false);
+  const [isSpeaking] = useState(false);
+  const [isLoading, setIsLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  const streamRef = useRef<MediaStream | null>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
+  const workletRef = useRef<AudioWorkletNode | null>(null);
+  const scriptRef = useRef<ScriptProcessorNode | null>(null);
+  const onAudioDataRef = useRef(onAudioData);
+  const isMutedRef = useRef(isMuted);
+  const pendingRef = useRef<number[]>([]);
+  const isMountedRef = useRef(true);
+
+  useEffect(() => {
+    onAudioDataRef.current = onAudioData;
+  }, [onAudioData]);
+
+  useEffect(() => {
+    isMutedRef.current = isMuted;
+  }, [isMuted]);
+
+  const setMuted = useCallback((muted: boolean) => {
+    isMutedRef.current = muted;
+  }, []);
+
+  // Convert a Float32 frame at the source sample rate to 16 kHz Int16 chunks and
+  // emit them via onAudioData. Frames accumulate until we have at least
+  // CHUNK_SAMPLES at 16 kHz, then flush.
+  const handleFrame = useCallback((frame: Float32Array, fromRate: number) => {
+    if (isMutedRef.current) return;
+    const cb = onAudioDataRef.current;
+    if (!cb) return;
+
+    const downsampled = fromRate === TARGET_SAMPLE_RATE ? frame : downsampleFloat32(frame, fromRate, TARGET_SAMPLE_RATE);
+
+    const pending = pendingRef.current;
+    for (let i = 0; i < downsampled.length; i++) pending.push(downsampled[i]);
+
+    while (pending.length >= CHUNK_SAMPLES) {
+      const chunk = new Float32Array(pending.splice(0, CHUNK_SAMPLES));
+      const int16 = float32ToInt16(chunk);
+      const buf = int16.buffer.slice(int16.byteOffset, int16.byteOffset + int16.byteLength) as ArrayBuffer;
+      cb(buf);
+    }
+  }, []);
+
+  const teardownGraph = useCallback(async () => {
+    try {
+      if (workletRef.current) {
+        workletRef.current.port.onmessage = null;
+        workletRef.current.disconnect();
+        workletRef.current = null;
+      }
+      if (scriptRef.current) {
+        scriptRef.current.onaudioprocess = null;
+        scriptRef.current.disconnect();
+        scriptRef.current = null;
+      }
+      if (sourceRef.current) {
+        sourceRef.current.disconnect();
+        sourceRef.current = null;
+      }
+      if (streamRef.current) {
+        streamRef.current.getTracks().forEach((t) => t.stop());
+        streamRef.current = null;
+      }
+      if (audioContextRef.current && audioContextRef.current.state !== "closed") {
+        await audioContextRef.current.close();
+      }
+      audioContextRef.current = null;
+      pendingRef.current = [];
+    } catch (err) {
+      console.warn("[AudioCapture] teardown error:", err);
+    }
+  }, []);
+
+  const startCapture = useCallback(async (): Promise<{ success: boolean; reason?: string }> => {
+    if (isCapturing || streamRef.current) {
+      return { success: true };
+    }
+
+    setIsLoading(true);
+    setError(null);
+
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
+      });
+      streamRef.current = stream;
+
+      const AudioContextClass = (window as unknown as { AudioContext?: typeof AudioContext; webkitAudioContext?: typeof AudioContext }).AudioContext ||
+        (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
+      const audioContext: AudioContext = new (AudioContextClass as typeof AudioContext)();
+      audioContextRef.current = audioContext;
+      const fromRate = audioContext.sampleRate;
+
+      const source = audioContext.createMediaStreamSource(stream);
+      sourceRef.current = source;
+
+      // Prefer AudioWorklet (modern). Fall back to ScriptProcessor.
+      let usedWorklet = false;
+      if (audioContext.audioWorklet) {
+        try {
+          const workletCode = `
+            class CaptureProcessor extends AudioWorkletProcessor {
+              process(inputs) {
+                const ch = inputs[0] && inputs[0][0];
+                if (ch && ch.length) {
+                  this.port.postMessage(new Float32Array(ch));
+                }
+                return true;
+              }
+            }
+            registerProcessor('gravity-capture-processor', CaptureProcessor);
+          `;
+          const blob = new Blob([workletCode], { type: "application/javascript" });
+          const url = URL.createObjectURL(blob);
+          await audioContext.audioWorklet.addModule(url);
+          URL.revokeObjectURL(url);
+
+          const node = new AudioWorkletNode(audioContext, "gravity-capture-processor");
+          node.port.onmessage = (ev: MessageEvent<Float32Array>) => {
+            handleFrame(ev.data, fromRate);
+          };
+          source.connect(node);
+          // The worklet must connect to destination to keep ticking; route through a
+          // muted gain so the mic is not played back.
+          const sink = audioContext.createGain();
+          sink.gain.value = 0;
+          node.connect(sink).connect(audioContext.destination);
+          workletRef.current = node;
+          usedWorklet = true;
+        } catch (err) {
+          console.warn("[AudioCapture] AudioWorklet unavailable, falling back to ScriptProcessor:", err);
+        }
+      }
+
+      if (!usedWorklet) {
+        const bufferSize = 4096;
+        const node = audioContext.createScriptProcessor(bufferSize, 1, 1);
+        node.onaudioprocess = (ev) => {
+          const ch = ev.inputBuffer.getChannelData(0);
+          handleFrame(new Float32Array(ch), fromRate);
+        };
+        source.connect(node);
+        const sink = audioContext.createGain();
+        sink.gain.value = 0;
+        node.connect(sink).connect(audioContext.destination);
+        scriptRef.current = node;
+      }
+
+      if (isMountedRef.current) {
+        setIsCapturing(true);
+        setIsLoading(false);
+      }
+      return { success: true };
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.error("[AudioCapture] Failed to start:", err);
+      if (isMountedRef.current) {
+        setError(msg);
+        setIsLoading(false);
+      }
+      await teardownGraph();
+      return { success: false, reason: msg };
+    }
+  }, [isCapturing, handleFrame, teardownGraph]);
+
+  const stopCapture = useCallback(async (): Promise<{ success: boolean; reason?: string }> => {
+    if (!streamRef.current && !audioContextRef.current) {
+      return { success: false, reason: "not_running" };
+    }
+    await teardownGraph();
+    if (isMountedRef.current) {
+      setIsCapturing(false);
+    }
+    return { success: true };
+  }, [teardownGraph]);
+
+  const toggleCapture = useCallback(async () => {
+    if (isCapturing) return await stopCapture();
+    return await startCapture();
+  }, [isCapturing, startCapture, stopCapture]);
+
+  useEffect(() => {
+    return () => {
+      isMountedRef.current = false;
+      teardownGraph();
+    };
+  }, [teardownGraph]);
+
+  return { startCapture, stopCapture, toggleCapture, setMuted, isCapturing, isSpeaking, isLoading, error };
+}

package/vendor/sdk/lib/realtime/useAudioPlayback.ts

@@ -0,0 +1,145 @@
+/**
+ * useAudioPlayback — LPCM audio playback with timeline-based scheduling.
+ *
+ * Web Audio API for low-latency playback and accurate timing. Audio format from
+ * the server (Nova / Grok): LPCM Int16, 24 kHz, mono.
+ *
+ * Ported from gravity-client/src/realtime/useAudioPlayback.ts (the proven, live
+ * implementation), trimmed to the surface the `voice` service uses.
+ */
+
+import { useRef, useCallback, useEffect, useState } from "react";
+
+/** Decode LPCM (Int16, 24 kHz, mono) → AudioBuffer. */
+function decodeLPCM(audioContext: AudioContext, data: ArrayBuffer): AudioBuffer {
+  const sampleRate = 24000;
+  const int16 = new Int16Array(data);
+  const numSamples = int16.length;
+  const audioBuffer = audioContext.createBuffer(1, numSamples, sampleRate);
+  const channelData = audioBuffer.getChannelData(0);
+  for (let i = 0; i < numSamples; i++) channelData[i] = int16[i] / 32768;
+  return audioBuffer;
+}
+
+export interface UseAudioPlaybackReturn {
+  playAudio: (audioData: ArrayBuffer) => void;
+  stopAll: () => void;
+  isPlaying: boolean;
+  /** Mark that no more chunks are coming — isPlaying clears when the last finishes. */
+  markAsLastChunk: () => void;
+}
+
+interface QueuedAudio {
+  audioData: ArrayBuffer;
+}
+
+export function useAudioPlayback(): UseAudioPlaybackReturn {
+  const [isPlaying, setIsPlaying] = useState(false);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const gainNodeRef = useRef<GainNode | null>(null);
+  const queueRef = useRef<QueuedAudio[]>([]);
+  const nextTimeRef = useRef<number>(0);
+  const isProcessingRef = useRef(false);
+  const activeSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
+  const hasStartedRef = useRef(false);
+  const isLastChunkRef = useRef(false);
+
+  const getAudioContext = useCallback(() => {
+    if (!audioContextRef.current) {
+      audioContextRef.current = new AudioContext({ sampleRate: 24000 });
+      gainNodeRef.current = audioContextRef.current.createGain();
+      gainNodeRef.current.connect(audioContextRef.current.destination);
+    }
+    if (audioContextRef.current.state === "suspended") {
+      audioContextRef.current.resume();
+    }
+    return audioContextRef.current;
+  }, []);
+
+  const processQueue = useCallback(async () => {
+    if (isProcessingRef.current) return;
+    if (queueRef.current.length === 0) return;
+
+    isProcessingRef.current = true;
+    const audioContext = getAudioContext();
+    const gainNode = gainNodeRef.current!;
+
+    while (queueRef.current.length > 0) {
+      const item = queueRef.current.shift()!;
+      try {
+        const audioBuffer = decodeLPCM(audioContext, item.audioData);
+        const now = audioContext.currentTime;
+        const startTime = Math.max(now, nextTimeRef.current);
+
+        const source = audioContext.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(gainNode);
+        activeSourcesRef.current.add(source);
+
+        if (!hasStartedRef.current) {
+          hasStartedRef.current = true;
+          setIsPlaying(true);
+        }
+
+        source.start(startTime);
+        nextTimeRef.current = startTime + audioBuffer.duration;
+
+        source.onended = () => {
+          activeSourcesRef.current.delete(source);
+          if (isLastChunkRef.current && activeSourcesRef.current.size === 0 && queueRef.current.length === 0) {
+            setIsPlaying(false);
+            hasStartedRef.current = false;
+            isLastChunkRef.current = false;
+          }
+        };
+      } catch (error) {
+        console.error("[AudioPlayback] Failed to decode audio:", error);
+      }
+    }
+
+    isProcessingRef.current = false;
+  }, [getAudioContext]);
+
+  const playAudio = useCallback(
+    (audioData: ArrayBuffer) => {
+      queueRef.current.push({ audioData });
+      processQueue();
+    },
+    [processQueue],
+  );
+
+  const stopAll = useCallback(() => {
+    activeSourcesRef.current.forEach((source) => {
+      try {
+        source.stop();
+      } catch {
+        // already stopped
+      }
+    });
+    activeSourcesRef.current.clear();
+    queueRef.current = [];
+    nextTimeRef.current = 0;
+    hasStartedRef.current = false;
+    isProcessingRef.current = false;
+    isLastChunkRef.current = false;
+    setIsPlaying(false);
+  }, []);
+
+  const markAsLastChunk = useCallback(() => {
+    isLastChunkRef.current = true;
+    if (activeSourcesRef.current.size === 0 && queueRef.current.length === 0 && hasStartedRef.current) {
+      setIsPlaying(false);
+      hasStartedRef.current = false;
+      isLastChunkRef.current = false;
+    }
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      stopAll();
+      if (audioContextRef.current) audioContextRef.current.close();
+    };
+  }, [stopAll]);
+
+  return { playAudio, stopAll, isPlaying, markAsLastChunk };
+}

package/vendor/sdk/lib/realtime/useRealtimeWebSocket.ts

@@ -0,0 +1,164 @@
+/**
+ * useRealtimeWebSocket — binary WebSocket for real-time audio streaming.
+ *
+ * Bidirectional audio over the `/ws/gravity` lane: sends binary PCM Int16 16 kHz
+ * up, receives binary LPCM Int16 24 kHz back, plus JSON control messages for
+ * audio-state events. This is the ONLY remaining WS use (docs/VOICE_STREAMING_GUIDE);
+ * the audio-state control events ride this lane WITH the frames so they stay
+ * frame-synchronized — they never go on the MCP component stream.
+ *
+ * Ported verbatim from gravity-client/src/realtime/useRealtimeWebSocket.ts — it is
+ * already self-contained (no dependency on the legacy unified client), which is why
+ * the Unoverse `voice` service can own it directly.
+ */
+
+import { useCallback, useRef, useState, useEffect } from "react";
+import type { ControlMessage, AudioStateEvent } from "./types";
+
+export interface UseRealtimeWebSocketOptions {
+  /** Conversation ID for the WebSocket session. */
+  sessionId: string;
+  userId?: string;
+  chatId?: string;
+  /** WebSocket base URL (e.g. ws://localhost:4100). Defaults to current origin. */
+  wsUrl?: string;
+  /** Function to get a fresh access token (called on each connect). */
+  getAccessToken?: () => Promise<string | null> | string | null;
+  onAudioReceived?: (audioData: ArrayBuffer) => void;
+  onConnectionChange?: (connected: boolean) => void;
+  onControlMessage?: (message: ControlMessage) => void;
+  onAudioState?: (event: AudioStateEvent) => void;
+}
+
+export interface UseRealtimeWebSocketReturn {
+  isConnected: boolean;
+  sendAudio: (audioData: ArrayBuffer) => void;
+  sendControl: (type: string, data?: Record<string, unknown>) => void;
+  connect: () => Promise<void>;
+  disconnect: () => void;
+}
+
+export function useRealtimeWebSocket(options: UseRealtimeWebSocketOptions): UseRealtimeWebSocketReturn {
+  const { sessionId, userId, chatId, wsUrl, getAccessToken, onAudioReceived, onConnectionChange, onControlMessage, onAudioState } = options;
+
+  const wsRef = useRef<WebSocket | null>(null);
+  const [isConnected, setIsConnected] = useState(false);
+
+  const getWsUrl = useCallback(async () => {
+    let baseUrl: string;
+    if (wsUrl) {
+      baseUrl = `${wsUrl}/ws/gravity`;
+    } else {
+      const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
+      baseUrl = `${protocol}//${window.location.host}/ws/gravity`;
+    }
+    let token: string | null = null;
+    if (getAccessToken) {
+      try {
+        token = await getAccessToken();
+      } catch (error) {
+        console.warn("[RealtimeWebSocket] Failed to get access token:", error);
+      }
+    }
+    return token ? `${baseUrl}?token=${encodeURIComponent(token)}` : baseUrl;
+  }, [wsUrl, getAccessToken]);
+
+  const connect = useCallback(async (): Promise<void> => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) return;
+    if (wsRef.current?.readyState === WebSocket.CONNECTING) {
+      return new Promise((resolve) => {
+        const checkInterval = setInterval(() => {
+          if (wsRef.current?.readyState === WebSocket.OPEN) {
+            clearInterval(checkInterval);
+            resolve();
+          }
+        }, 50);
+      });
+    }
+
+    const url = await getWsUrl();
+    console.log("[RealtimeWebSocket] Connecting to:", url.replace(/token=[^&]+/, "token=***"));
+
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(url);
+      ws.binaryType = "arraybuffer";
+
+      const timeout = setTimeout(() => reject(new Error("WebSocket connection timeout")), 10000);
+
+      ws.onopen = () => {
+        clearTimeout(timeout);
+        const initMessage = {
+          type: "INIT_SESSION",
+          conversationId: sessionId,
+          userId: userId || "anonymous",
+          chatId: chatId || `chat_${sessionId}`,
+        };
+        ws.send(JSON.stringify(initMessage));
+        setIsConnected(true);
+        onConnectionChange?.(true);
+        resolve();
+      };
+
+      ws.onmessage = (event) => {
+        if (event.data instanceof ArrayBuffer) {
+          onAudioReceived?.(event.data);
+        } else {
+          try {
+            const message: ControlMessage = JSON.parse(event.data as string);
+            onControlMessage?.(message);
+            // Only honour control-channel state events (`message.state`).
+            // `message.audioState` is per-chunk metadata on audio publishes.
+            if (message.state && onAudioState) {
+              onAudioState({ state: message.state, metadata: message.metadata });
+            }
+          } catch (error) {
+            console.error("[RealtimeWebSocket] Failed to parse control message:", error);
+          }
+        }
+      };
+
+      ws.onerror = (error) => {
+        clearTimeout(timeout);
+        console.error("[RealtimeWebSocket] Error:", error);
+        reject(error);
+      };
+
+      ws.onclose = () => {
+        setIsConnected(false);
+        onConnectionChange?.(false);
+      };
+
+      wsRef.current = ws;
+    });
+  }, [getWsUrl, sessionId, userId, chatId, onAudioReceived, onConnectionChange, onControlMessage, onAudioState]);
+
+  const disconnect = useCallback(() => {
+    if (wsRef.current) {
+      wsRef.current.close();
+      wsRef.current = null;
+    }
+  }, []);
+
+  const sendAudio = useCallback((audioData: ArrayBuffer) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(audioData);
+    }
+  }, []);
+
+  const sendControl = useCallback((type: string, data?: Record<string, unknown>) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify({ type, ...data }));
+    }
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      if (wsRef.current) {
+        wsRef.current.close();
+        wsRef.current = null;
+      }
+    };
+  }, []);
+
+  return { isConnected, sendAudio, sendControl, connect, disconnect };
+}