@alia.onl/sdk 1.0.0 → 2.0.0

package/src/hooks/useAudioLevelMonitor.ts

@@ -0,0 +1,142 @@
+/**
+ * Real-time audio level monitoring using Web Audio API.
+ *
+ * Extracts RMS levels from LiveKit audio tracks (local mic + remote agents)
+ * and returns them at ~20fps for driving wave animations.
+ *
+ * Supports multiple remote audio tracks (primary + cohost) by taking
+ * the max RMS level across all subscribed remote tracks.
+ */
+
+import { useState, useEffect, useRef } from 'react';
+import { Room, RoomEvent, Track } from 'livekit-client';
+
+/** Compute RMS (root mean square) amplitude from an AnalyserNode. Returns 0-1. */
+function getRMS(analyser: AnalyserNode, buffer: any): number {
+  analyser.getFloatTimeDomainData(buffer);
+  let sum = 0;
+  for (let i = 0; i < buffer.length; i++) sum += buffer[i] * buffer[i];
+  return Math.min(1, Math.sqrt(sum / buffer.length) * 8);
+}
+
+export function useAudioLevelMonitor(room: Room | null, isConnected: boolean) {
+  const [captureLevel, setCaptureLevel] = useState(0);
+  const [playbackLevel, setPlaybackLevel] = useState(0);
+  const cleanupRef = useRef<(() => void) | null>(null);
+
+  useEffect(() => {
+    // Tear down previous monitors
+    cleanupRef.current?.();
+    cleanupRef.current = null;
+
+    if (!room || !isConnected) {
+      setCaptureLevel(0);
+      setPlaybackLevel(0);
+      return;
+    }
+
+    // Guard: Web Audio API required
+    if (typeof AudioContext === 'undefined') return;
+
+    const audioCtx = new AudioContext();
+    // Resume AudioContext (may be suspended until user gesture on mobile/WebView)
+    if (audioCtx.state === 'suspended') {
+      audioCtx.resume().catch(() => {});
+    }
+
+    let localAnalyser: AnalyserNode | null = null;
+    let localBuffer: Float32Array | null = null;
+    const remoteAnalysers = new Map<string, { analyser: AnalyserNode; buffer: Float32Array }>();
+    let rafId = 0;
+    let disposed = false;
+
+    // ---------- Local mic ----------
+    const localTrack = room.localParticipant?.getTrackPublication(Track.Source.Microphone)?.track;
+    if (localTrack?.mediaStreamTrack) {
+      try {
+        const source = audioCtx.createMediaStreamSource(new MediaStream([localTrack.mediaStreamTrack]));
+        localAnalyser = audioCtx.createAnalyser();
+        localAnalyser.fftSize = 256;
+        source.connect(localAnalyser);
+        localBuffer = new Float32Array(localAnalyser.fftSize);
+      } catch {
+        // MediaStream creation can fail in some environments
+      }
+    }
+
+    // ---------- Remote agent audio (supports multiple tracks) ----------
+    const setupRemoteAnalyser = (mediaStreamTrack: MediaStreamTrack, key: string) => {
+      if (remoteAnalysers.has(key) || disposed) return;
+      try {
+        const source = audioCtx.createMediaStreamSource(new MediaStream([mediaStreamTrack]));
+        const analyser = audioCtx.createAnalyser();
+        analyser.fftSize = 256;
+        source.connect(analyser);
+        const buffer = new Float32Array(analyser.fftSize);
+        remoteAnalysers.set(key, { analyser, buffer });
+      } catch {
+        // Ignore errors
+      }
+    };
+
+    // Check already-subscribed remote tracks
+    for (const [, p] of room.remoteParticipants) {
+      for (const [, pub] of p.audioTrackPublications) {
+        if (pub.track?.mediaStreamTrack) {
+          setupRemoteAnalyser(pub.track.mediaStreamTrack, `${p.identity}-${pub.trackSid}`);
+        }
+      }
+    }
+
+    // Listen for new remote audio tracks
+    const onTrackSubscribed = (track: any, publication: any, participant: any) => {
+      if (track.kind === Track.Kind.Audio && track.mediaStreamTrack) {
+        setupRemoteAnalyser(track.mediaStreamTrack, `${participant.identity}-${publication.trackSid}`);
+      }
+    };
+    room.on(RoomEvent.TrackSubscribed, onTrackSubscribed);
+
+    // Clean up stale analysers when tracks are unsubscribed
+    const onTrackUnsubscribed = (track: any, publication: any, participant: any) => {
+      const key = `${participant.identity}-${publication.trackSid}`;
+      remoteAnalysers.delete(key);
+    };
+    room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
+
+    // ---------- Poll loop (~20fps) ----------
+    let lastUpdate = 0;
+    const poll = (time: number) => {
+      if (disposed) return;
+      if (time - lastUpdate > 50) {
+        lastUpdate = time;
+        if (localAnalyser && localBuffer) {
+          setCaptureLevel(getRMS(localAnalyser, localBuffer));
+        }
+        // Take max RMS across all remote tracks (primary + cohost)
+        let maxRemoteLevel = 0;
+        for (const [, { analyser, buffer }] of remoteAnalysers) {
+          const level = getRMS(analyser, buffer);
+          if (level > maxRemoteLevel) maxRemoteLevel = level;
+        }
+        setPlaybackLevel(maxRemoteLevel);
+      }
+      rafId = requestAnimationFrame(poll);
+    };
+    rafId = requestAnimationFrame(poll);
+
+    // ---------- Cleanup ----------
+    const dispose = () => {
+      disposed = true;
+      cancelAnimationFrame(rafId);
+      room.off(RoomEvent.TrackSubscribed, onTrackSubscribed);
+      room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
+      remoteAnalysers.clear();
+      audioCtx.close().catch(() => {});
+    };
+    cleanupRef.current = dispose;
+
+    return dispose;
+  }, [room, isConnected]);
+
+  return { captureLevel, playbackLevel };
+}

package/src/hooks/useAudioLevels.ts

@@ -0,0 +1,51 @@
+import { useEffect } from 'react';
+import { useSharedValue, withTiming, Easing } from 'react-native-reanimated';
+import type { AgentState } from '../types';
+
+interface UseAudioLevelsOptions {
+  captureLevel: number;
+  playbackLevel: number;
+  agentState: AgentState;
+  isConnected: boolean;
+}
+
+export function useAudioLevels({
+  captureLevel,
+  playbackLevel,
+  agentState,
+  isConnected,
+}: UseAudioLevelsOptions) {
+  const waveAmplitude = useSharedValue(0);
+
+  useEffect(() => {
+    if (!isConnected) {
+      waveAmplitude.value = withTiming(0, { duration: 300 });
+      return;
+    }
+
+    const amplifiedPlayback = Math.min(1, playbackLevel * 3);
+
+    let target = 0;
+    if (agentState === 'thinking') {
+      target = Math.max(0.15, captureLevel);
+    } else if (agentState === 'speaking') {
+      // 0.4 baseline ensures visible waves while AI speaks
+      target = Math.max(0.4, amplifiedPlayback, captureLevel * 0.5);
+    } else if (agentState === 'listening') {
+      // Subtle pulse while listening; mic level takes over when available
+      target = Math.max(0.08, captureLevel, amplifiedPlayback * 0.5);
+    } else {
+      // Idle — calm, no minimum
+      target = Math.max(captureLevel, amplifiedPlayback * 0.5);
+    }
+
+    // Fast attack, slower decay for natural VU-meter feel
+    const duration = target > waveAmplitude.value ? 60 : 200;
+    waveAmplitude.value = withTiming(target, {
+      duration,
+      easing: Easing.bezier(0.33, 1, 0.68, 1),
+    });
+  }, [captureLevel, playbackLevel, agentState, isConnected]);
+
+  return { waveAmplitude };
+}

package/src/hooks/useSoundEffects.ts

@@ -0,0 +1,128 @@
+/**
+ * Sound effects hook for voice mode.
+ * Plays short audio cues when the AI is thinking, using tools,
+ * or when voice connection state changes.
+ *
+ * Sounds are provided via options instead of hardcoded require() calls,
+ * making this hook portable across different bundlers and apps.
+ */
+
+import { useEffect, useRef, useCallback } from 'react';
+import type { AgentState } from '../types';
+
+export type SoundName = 'thinking' | 'toolCall' | 'voiceConnect' | 'voiceDisconnect';
+
+export interface SoundSources {
+  thinking?: any;
+  toolCall?: any;
+  voiceConnect?: any;
+  voiceDisconnect?: any;
+}
+
+const SOUND_NAMES: SoundName[] = ['thinking', 'toolCall', 'voiceConnect', 'voiceDisconnect'];
+
+export function useSoundEffects(enabled: boolean = false, sounds: SoundSources = {}) {
+  const playersRef = useRef<Map<SoundName, any>>(new Map());
+  const loadedRef = useRef(false);
+
+  useEffect(() => {
+    if (!enabled || loadedRef.current) return;
+
+    // Only load sounds that were actually provided
+    const hasAnySounds = SOUND_NAMES.some((name) => sounds[name] != null);
+    if (!hasAnySounds) return;
+
+    let mounted = true;
+
+    (async () => {
+      try {
+        const { createAudioPlayer } = await import('expo-audio');
+        if (!mounted) return;
+
+        for (const name of SOUND_NAMES) {
+          const source = sounds[name];
+          if (source == null) continue;
+          try {
+            const player = createAudioPlayer(source);
+            playersRef.current.set(name, player);
+          } catch {
+            // Individual sound failed to load — skip it
+          }
+        }
+        loadedRef.current = true;
+      } catch {
+        // expo-audio not available — degrade silently
+      }
+    })();
+
+    return () => {
+      mounted = false;
+      for (const player of playersRef.current.values()) {
+        try { player.release(); } catch {}
+      }
+      playersRef.current.clear();
+      loadedRef.current = false;
+    };
+  }, [enabled, sounds]);
+
+  const play = useCallback((sound: SoundName) => {
+    if (!enabled) return;
+    const player = playersRef.current.get(sound);
+    if (player) {
+      try {
+        player.seekTo(0);
+        player.play();
+      } catch {
+        // Playback failed — ignore
+      }
+    }
+  }, [enabled]);
+
+  return { play };
+}
+
+/**
+ * Hook that automatically triggers sound effects based on voice state changes.
+ * Wire this into the conversation page alongside useVoiceMode.
+ */
+export function useVoiceSoundEffects({
+  isVoiceActive,
+  agentState,
+  isConnected,
+  sounds = {},
+}: {
+  isVoiceActive: boolean;
+  agentState: AgentState;
+  isConnected: boolean;
+  sounds?: SoundSources;
+}) {
+  const { play } = useSoundEffects(isVoiceActive, sounds);
+  const prevAgentStateRef = useRef<AgentState>('idle');
+  const prevConnectedRef = useRef(false);
+
+  // Play sound on agent state transitions
+  useEffect(() => {
+    if (!isVoiceActive) return;
+
+    const prev = prevAgentStateRef.current;
+    prevAgentStateRef.current = agentState;
+
+    if (agentState === 'thinking' && prev !== 'thinking') {
+      play('thinking');
+    }
+  }, [agentState, isVoiceActive, play]);
+
+  // Play sound on connection state changes
+  useEffect(() => {
+    if (!isVoiceActive) return;
+
+    const wasConnected = prevConnectedRef.current;
+    prevConnectedRef.current = isConnected;
+
+    if (isConnected && !wasConnected) {
+      play('voiceConnect');
+    } else if (!isConnected && wasConnected) {
+      play('voiceDisconnect');
+    }
+  }, [isConnected, isVoiceActive, play]);
+}

package/src/hooks/useSpeechToText.ts

@@ -0,0 +1,180 @@
+import { useState, useCallback, useRef, useEffect } from 'react';
+import {
+  useAudioRecorder,
+  useAudioRecorderState,
+  RecordingPresets,
+  requestRecordingPermissionsAsync,
+  setAudioModeAsync,
+} from 'expo-audio';
+import { useOxy } from '@oxyhq/services';
+import { create } from 'zustand';
+
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+
+// ============== OPTIONS ==============
+
+export interface UseSTTOptions {
+  apiUrl?: string;
+  accessToken?: string;
+}
+
+// ============== INLINE STT STORE ==============
+
+interface STTStoreState {
+  isRecording: boolean;
+  metering: number;
+  setRecording: (v: boolean) => void;
+  setMetering: (v: number) => void;
+}
+
+const useSTTStore = create<STTStoreState>((set) => ({
+  isRecording: false,
+  metering: 0,
+  setRecording: (isRecording) => set({ isRecording }),
+  setMetering: (metering) => set({ metering }),
+}));
+
+// ============== HOOK ==============
+
+type STTState = 'idle' | 'recording' | 'transcribing';
+
+const METERING_PRESET = { ...RecordingPresets.HIGH_QUALITY, isMeteringEnabled: true };
+
+export function useSpeechToText(options: UseSTTOptions = {}) {
+  const apiUrl = options.apiUrl || API_URL;
+
+  const [state, setState] = useState<STTState>('idle');
+  const [error, setError] = useState<string | null>(null);
+  const isRecordingRef = useRef(false);
+  const { oxyServices } = useOxy();
+
+  const audioRecorder = useAudioRecorder(METERING_PRESET);
+  const recorderState = useAudioRecorderState(audioRecorder, 100);
+
+  const sttStore = useSTTStore;
+
+  // ============== AUTH ==============
+
+  const getToken = useCallback((): string | null => {
+    if (options.accessToken) return options.accessToken;
+    return oxyServices.httpService.getAccessToken();
+  }, [options.accessToken, oxyServices]);
+
+  // Push metering to store while recording
+  useEffect(() => {
+    if (state === 'recording' && recorderState.metering != null) {
+      // dBFS: -160 (silence) to 0 (max). Normalize using -60 as practical floor.
+      const normalized = Math.min(1, Math.max(0, (recorderState.metering + 60) / 60));
+      sttStore.getState().setMetering(normalized);
+    }
+  }, [state, recorderState.metering]);
+
+  // Sync recording state to store
+  useEffect(() => {
+    sttStore.getState().setRecording(state === 'recording');
+  }, [state]);
+
+  const startRecording = useCallback(async () => {
+    try {
+      setError(null);
+
+      const permStatus = await requestRecordingPermissionsAsync();
+      if (!permStatus.granted) {
+        setError('Microphone permission required');
+        return;
+      }
+
+      setState('recording');
+
+      await setAudioModeAsync({
+        allowsRecording: true,
+        playsInSilentMode: true,
+      });
+
+      await audioRecorder.prepareToRecordAsync();
+      audioRecorder.record();
+      isRecordingRef.current = true;
+    } catch (e: any) {
+      console.error('[STT] Recording error:', e);
+      setError('Failed to start recording');
+      setState('idle');
+      isRecordingRef.current = false;
+    }
+  }, [audioRecorder]);
+
+  const stopAndTranscribe = useCallback(async (): Promise<string | null> => {
+    if (!isRecordingRef.current) return null;
+
+    try {
+      setState('transcribing');
+      isRecordingRef.current = false;
+      await audioRecorder.stop();
+      const uri = audioRecorder.uri;
+
+      if (!uri) {
+        setState('idle');
+        return null;
+      }
+
+      // Read audio file as base64
+      const response = await fetch(uri);
+      const blob = await response.blob();
+      const base64 = await new Promise<string>((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => {
+          const result = reader.result as string;
+          resolve(result.split(',')[1]);
+        };
+        reader.onerror = reject;
+        reader.readAsDataURL(blob);
+      });
+
+      // Send to transcription API
+      const token = getToken();
+      const transcribeResponse = await fetch(`${apiUrl}/v1/voice/transcribe`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify({
+          audio: base64,
+          format: 'audio/m4a',
+        }),
+      });
+
+      if (!transcribeResponse.ok) {
+        const errorData = await transcribeResponse.json().catch(() => ({}));
+        throw new Error(errorData.error || 'Transcription failed');
+      }
+
+      const result = await transcribeResponse.json() as { text: string };
+      setState('idle');
+      return result.text || null;
+    } catch (e: any) {
+      console.error('[STT] Transcription error:', e);
+      setError(e.message || 'Transcription failed');
+      setState('idle');
+      return null;
+    }
+  }, [audioRecorder, getToken, apiUrl]);
+
+  const cancel = useCallback(() => {
+    if (isRecordingRef.current) {
+      isRecordingRef.current = false;
+      Promise.resolve(audioRecorder.stop()).catch(() => {});
+    }
+    setState('idle');
+    setError(null);
+  }, [audioRecorder]);
+
+  return {
+    state,
+    error,
+    startRecording,
+    stopAndTranscribe,
+    cancel,
+    isRecording: state === 'recording',
+    isTranscribing: state === 'transcribing',
+  };
+}