@alia.onl/sdk 1.0.0 → 2.0.0

package/src/hooks/useTTS.ts

@@ -0,0 +1,233 @@
+import { useCallback, useEffect, useRef } from 'react';
+import {
+  useSharedValue,
+  withRepeat,
+  withSequence,
+  withTiming,
+  cancelAnimation,
+  Easing,
+} from 'react-native-reanimated';
+import { useOxy } from '@oxyhq/services';
+import { create } from 'zustand';
+
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+
+// ============== OPTIONS ==============
+
+export interface UseTTSOptions {
+  apiUrl?: string;
+  accessToken?: string;
+  voice?: 'male' | 'female';
+  tone?: 'brief' | 'chill' | 'default';
+}
+
+// ============== INLINE TTS STORE ==============
+
+type PlaybackState = 'idle' | 'loading' | 'playing' | 'paused' | 'error';
+
+interface TTSStore {
+  activeMessageId: string | null;
+  playbackState: PlaybackState;
+  error: string | null;
+  setActiveMessage: (id: string | null) => void;
+  setPlaybackState: (state: PlaybackState) => void;
+  setError: (error: string | null) => void;
+  reset: () => void;
+}
+
+const useTTSStore = create<TTSStore>((set) => ({
+  activeMessageId: null,
+  playbackState: 'idle',
+  error: null,
+  setActiveMessage: (id) => set({ activeMessageId: id }),
+  setPlaybackState: (playbackState) => set({ playbackState }),
+  setError: (error) => set({ error, playbackState: error ? 'error' : 'idle' }),
+  reset: () => set({ activeMessageId: null, playbackState: 'idle', error: null }),
+}));
+
+// ============== HOOK ==============
+
+export function useTTS(options: UseTTSOptions = {}) {
+  const apiUrl = options.apiUrl || API_URL;
+  const voicePref = options.voice ?? 'female';
+  const tonePref = options.tone ?? 'default';
+
+  const { oxyServices } = useOxy();
+  const {
+    activeMessageId,
+    playbackState,
+    error,
+    setActiveMessage,
+    setPlaybackState,
+    setError,
+    reset,
+  } = useTTSStore();
+
+  const playerRef = useRef<any>(null);
+
+  // Simulated wave amplitude for visualization
+  const ttsWaveAmplitude = useSharedValue(0);
+
+  // ============== AUTH ==============
+
+  const getToken = useCallback((): string | null => {
+    if (options.accessToken) return options.accessToken;
+    return oxyServices.httpService.getAccessToken();
+  }, [options.accessToken, oxyServices]);
+
+  // Animate wave when playing
+  useEffect(() => {
+    if (playbackState === 'playing') {
+      ttsWaveAmplitude.value = withRepeat(
+        withSequence(
+          withTiming(0.5, { duration: 800, easing: Easing.inOut(Easing.sin) }),
+          withTiming(0.2, { duration: 600, easing: Easing.inOut(Easing.sin) }),
+          withTiming(0.6, { duration: 700, easing: Easing.inOut(Easing.sin) }),
+          withTiming(0.15, { duration: 500, easing: Easing.inOut(Easing.sin) }),
+        ),
+        -1,
+      );
+    } else {
+      cancelAnimation(ttsWaveAmplitude);
+      ttsWaveAmplitude.value = withTiming(0, { duration: 300 });
+    }
+  }, [playbackState]);
+
+  const getTTSVoice = useCallback(() => {
+    return voicePref === 'male' ? 'echo' : 'nova';
+  }, [voicePref]);
+
+  const getTTSSpeed = useCallback(() => {
+    if (tonePref === 'brief') return 1.15;
+    if (tonePref === 'chill') return 0.9;
+    return 1.0;
+  }, [tonePref]);
+
+  const releasePlayer = useCallback(() => {
+    try {
+      playerRef.current?.remove();
+    } catch {}
+    playerRef.current = null;
+  }, []);
+
+  const stop = useCallback(() => {
+    releasePlayer();
+    reset();
+  }, [reset, releasePlayer]);
+
+  const playFromUrl = useCallback((audioUrl: string, _messageId: string) => {
+    releasePlayer();
+
+    (async () => {
+      try {
+        const { createAudioPlayer } = await import('expo-audio');
+        const player = createAudioPlayer({ uri: audioUrl });
+        playerRef.current = player;
+
+        player.addListener('playbackStatusUpdate', (status: any) => {
+          if (status.didJustFinish) {
+            releasePlayer();
+            reset();
+          }
+        });
+
+        player.play();
+        setPlaybackState('playing');
+      } catch {
+        setError('Audio playback not available');
+      }
+    })();
+  }, [releasePlayer, reset, setPlaybackState, setError]);
+
+  const readAloud = useCallback(async (
+    messageId: string,
+    text: string,
+    conversationId?: string,
+    audioUrl?: string,
+  ) => {
+    // If same message is playing, toggle pause/play
+    if (activeMessageId === messageId) {
+      if (playbackState === 'playing') {
+        playerRef.current?.pause();
+        setPlaybackState('paused');
+        return;
+      }
+      if (playbackState === 'paused') {
+        playerRef.current?.play();
+        setPlaybackState('playing');
+        return;
+      }
+    }
+
+    // Stop any current playback
+    if (activeMessageId) {
+      stop();
+    }
+
+    try {
+      setActiveMessage(messageId);
+      setPlaybackState('loading');
+
+      // If cached audioUrl exists, play directly
+      if (audioUrl) {
+        playFromUrl(audioUrl, messageId);
+        return;
+      }
+
+      // Call backend TTS API
+      const token = getToken();
+      if (!token) {
+        throw new Error('Not authenticated');
+      }
+
+      const response = await fetch(`${apiUrl}/v1/audio/speech`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify({
+          model: 'alia-v1-voice',
+          input: text,
+          voice: getTTSVoice(),
+          speed: getTTSSpeed(),
+          conversationId,
+          messageId,
+        }),
+      });
+
+      if (!response.ok) {
+        if (response.status === 504) {
+          throw new Error('Request timed out — please try again');
+        }
+        const errData = await response.json().catch(() => ({}));
+        throw new Error(errData.error?.message || errData.error || 'TTS failed');
+      }
+
+      const data = await response.json();
+      playFromUrl(data.audioUrl, messageId);
+    } catch (e: any) {
+      console.error('[TTS] Error:', e);
+      setError(e.message || 'Failed to read aloud');
+    }
+  }, [activeMessageId, playbackState, getToken, apiUrl, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      releasePlayer();
+    };
+  }, []);
+
+  return {
+    readAloud,
+    stop,
+    activeMessageId,
+    playbackState,
+    error,
+    ttsWaveAmplitude,
+    isPlaying: playbackState === 'playing',
+    isPaused: playbackState === 'paused',
+    isLoading: playbackState === 'loading',
+  };
+}

package/src/hooks/useVoiceRoom.ts

@@ -0,0 +1,420 @@
+/**
+ * Hook for real-time voice conversations via LiveKit rooms.
+ *
+ * Connects to a LiveKit room (created by POST /v1/voice/token),
+ * publishes the user's microphone, and receives agent audio + data
+ * messages (transcripts, state, cohost events) over WebRTC.
+ */
+
+import { useState, useCallback, useRef, useEffect } from 'react';
+import {
+  Room,
+  RoomEvent,
+  Track,
+  DisconnectReason,
+  type RemoteTrackPublication,
+  type RemoteParticipant,
+  type DataPublishOptions,
+} from 'livekit-client';
+import { useOxy } from '@oxyhq/services';
+import type { RoomState, AgentState, VoiceMessage, VoiceToolInvocation } from '../types';
+
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+
+// ============== OPTIONS ==============
+
+export interface UseVoiceRoomOptions {
+  apiUrl?: string;
+  voicePreference?: 'male' | 'female';
+  accessToken?: string;
+}
+
+// ============== TITLE TAG UTILITIES ==============
+
+const TAG = String.raw`ALIA_TITLE|TITLE|TÍTULO|TITRE|TITOLO|TITEL|ЗАГОЛОВОК`;
+
+const TITLE_STRIP_RE = new RegExp(
+  String.raw`\[(${TAG})\].*?\[\/\1\]|<(${TAG})>.*?<\/\2>`, 'gi',
+);
+
+const TITLE_PARTIAL_RE = new RegExp(
+  String.raw`\[(${TAG})\].*?(\[\/\1\])?$|<(${TAG})>.*?(<\/\3>)?$`, 'si',
+);
+
+function stripTitleTags(content: string): string {
+  return content.replace(TITLE_STRIP_RE, '').trim();
+}
+
+function stripTitleTagsPartial(content: string): string {
+  return content.replace(TITLE_STRIP_RE, '').replace(TITLE_PARTIAL_RE, '').trim();
+}
+
+// ============== INTERNAL DATA MESSAGE TYPES ==============
+
+interface AgentStateMsg { type: 'agent.state'; state: 'listening' | 'thinking' | 'speaking'; speaker: 'primary' | 'cohost' }
+interface TranscriptDeltaMsg { type: 'transcript.delta'; delta: string; speaker: 'primary' | 'cohost' }
+interface TranscriptDoneMsg { type: 'transcript.done'; transcript: string; speaker: 'primary' | 'cohost' }
+interface TranscriptUserMsg { type: 'transcript.user'; transcript: string }
+interface CohostEnabledMsg { type: 'cohost.enabled' }
+interface CohostDisabledMsg { type: 'cohost.disabled' }
+interface CohostTurnMsg { type: 'cohost.turn_changed'; speaker: 'primary' | 'cohost' | 'user' }
+interface CohostRoundMsg { type: 'cohost.round_complete'; turns: number }
+interface ToolCallMsg { type: 'tool.call'; toolName: string; callId: string; args?: any; speaker: 'primary' | 'cohost' }
+interface ToolResultMsg { type: 'tool.result'; callId: string; speaker: 'primary' | 'cohost' }
+interface SessionEndedMsg { type: 'session.ended'; reason: string }
+interface ErrorMsg { type: 'error'; code: string; message: string }
+
+type AgentDataMessage =
+  | AgentStateMsg | TranscriptDeltaMsg | TranscriptDoneMsg | TranscriptUserMsg
+  | CohostEnabledMsg | CohostDisabledMsg | CohostTurnMsg | CohostRoundMsg
+  | ToolCallMsg | ToolResultMsg | SessionEndedMsg | ErrorMsg;
+
+// ============== HOOK ==============
+
+export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
+  const apiUrl = options.apiUrl || API_URL;
+  const voicePref = options.voicePreference ?? 'female';
+
+  const [roomState, setRoomState] = useState<RoomState>('disconnected');
+  const [agentState, setAgentState] = useState<AgentState>('idle');
+  const [isMuted, setIsMuted] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [messages, setMessages] = useState<VoiceMessage[]>([]);
+  const [cohostActive, setCohostActive] = useState(false);
+  const [currentSpeaker, setCurrentSpeaker] = useState<'primary' | 'cohost' | 'user' | null>(null);
+  const [roundComplete, setRoundComplete] = useState(false);
+
+  const roomRef = useRef<Room | null>(null);
+  const sessionIdRef = useRef<string | null>(null);
+  const msgIdRef = useRef(0);
+  const sessionPrefixRef = useRef(`vm-${Date.now().toString(36)}`);
+
+  // Per-speaker streaming transcript refs
+  const primaryTextRef = useRef('');
+  const cohostTextRef = useRef('');
+
+  const { oxyServices } = useOxy();
+
+  // ============== AUTH ==============
+
+  const getToken = useCallback((): string | null => {
+    if (options.accessToken) return options.accessToken;
+    return oxyServices.httpService.getAccessToken();
+  }, [options.accessToken, oxyServices]);
+
+  // ============== CLEANUP ==============
+
+  const cleanup = useCallback(() => {
+    if (roomRef.current) {
+      // Detach all remote audio tracks before disconnecting
+      roomRef.current.remoteParticipants.forEach((p) => {
+        p.trackPublications.forEach((pub) => {
+          pub.track?.detach();
+        });
+      });
+      roomRef.current.disconnect();
+      roomRef.current = null;
+    }
+    sessionIdRef.current = null;
+    primaryTextRef.current = '';
+    cohostTextRef.current = '';
+  }, []);
+
+  // ============== DATA MESSAGE HANDLER ==============
+
+  const handleDataMessage = useCallback((payload: Uint8Array) => {
+    try {
+      const msg: AgentDataMessage = JSON.parse(new TextDecoder().decode(payload));
+
+      switch (msg.type) {
+        case 'agent.state': {
+          // Always update agentState so both primary and cohost drive wave animation
+          setAgentState(msg.state);
+          setCurrentSpeaker(msg.speaker);
+          break;
+        }
+
+        case 'transcript.delta': {
+          const textRef = msg.speaker === 'cohost' ? cohostTextRef : primaryTextRef;
+          textRef.current += msg.delta;
+          const text = stripTitleTagsPartial(textRef.current);
+          setMessages(prev => {
+            const last = prev[prev.length - 1];
+            if (last && last.role === 'assistant' && last.isStreaming && last.speaker === msg.speaker) {
+              return [...prev.slice(0, -1), { ...last, content: text }];
+            }
+            msgIdRef.current++;
+            return [...prev, {
+              id: `${sessionPrefixRef.current}-${msgIdRef.current}`,
+              role: 'assistant',
+              speaker: msg.speaker,
+              content: text,
+              timestamp: Date.now(),
+              isStreaming: true,
+            }];
+          });
+          break;
+        }
+
+        case 'transcript.done': {
+          const textRef = msg.speaker === 'cohost' ? cohostTextRef : primaryTextRef;
+          textRef.current = '';
+          const clean = stripTitleTags(msg.transcript);
+          setMessages(prev => {
+            const last = prev[prev.length - 1];
+            if (last && last.role === 'assistant' && last.isStreaming && last.speaker === msg.speaker) {
+              return [...prev.slice(0, -1), { ...last, content: clean, isStreaming: false }];
+            }
+            return prev;
+          });
+          break;
+        }
+
+        case 'transcript.user': {
+          if (!msg.transcript.trim()) break;
+          const transcript = msg.transcript.trim();
+          setMessages(prev => {
+            const last = prev[prev.length - 1];
+            // Update existing user message instead of creating duplicates
+            if (last && last.role === 'user') {
+              return [...prev.slice(0, -1), { ...last, content: transcript }];
+            }
+            msgIdRef.current++;
+            return [...prev, {
+              id: `${sessionPrefixRef.current}-${msgIdRef.current}`,
+              role: 'user',
+              content: transcript,
+              timestamp: Date.now(),
+              isStreaming: false,
+            }];
+          });
+          break;
+        }
+
+        case 'cohost.enabled':
+          setCohostActive(true);
+          break;
+
+        case 'cohost.disabled':
+          setCohostActive(false);
+          setCurrentSpeaker(null);
+          break;
+
+        case 'cohost.turn_changed':
+          setCurrentSpeaker(msg.speaker);
+          break;
+
+        case 'cohost.round_complete':
+          setRoundComplete(true);
+          break;
+
+        case 'tool.call': {
+          const tool: VoiceToolInvocation = { toolCallId: msg.callId, toolName: msg.toolName, state: 'call', args: msg.args };
+          setMessages(prev => {
+            const last = prev[prev.length - 1];
+            if (last && last.role === 'assistant' && last.speaker === msg.speaker) {
+              return [...prev.slice(0, -1), {
+                ...last,
+                toolInvocations: [...(last.toolInvocations || []), tool],
+              }];
+            }
+            msgIdRef.current++;
+            return [...prev, {
+              id: `${sessionPrefixRef.current}-${msgIdRef.current}`,
+              role: 'assistant' as const,
+              speaker: msg.speaker,
+              content: '',
+              timestamp: Date.now(),
+              isStreaming: true,
+              toolInvocations: [tool],
+            }];
+          });
+          break;
+        }
+
+        case 'tool.result': {
+          setMessages(prev => prev.map(m => {
+            if (!m.toolInvocations) return m;
+            const updated = m.toolInvocations.map(t =>
+              t.toolCallId === msg.callId ? { ...t, state: 'result' as const } : t
+            );
+            return { ...m, toolInvocations: updated };
+          }));
+          break;
+        }
+
+        case 'session.ended': {
+          cleanup();
+          setRoomState('disconnected');
+          setAgentState('idle');
+          const reason = msg.reason;
+          setError(
+            reason === 'user_silent' ? 'Call ended due to inactivity'
+            : reason === 'user_unresponsive' ? 'Call ended — no response'
+            : reason === 'max_duration_exceeded' ? 'Voice minutes limit reached. Upgrade for more.'
+            : reason === 'credits_exhausted' ? 'Not enough credits to continue. Add more or upgrade your plan.'
+            : 'Voice session ended'
+          );
+          break;
+        }
+
+        case 'error':
+          setError(msg.message);
+          break;
+      }
+    } catch {
+      // Ignore non-JSON data
+    }
+  }, [cleanup]);
+
+  // ============== CONNECT ==============
+
+  const connect = useCallback(async () => {
+    try {
+      setError(null);
+      setRoomState('connecting');
+
+      const token = getToken();
+      if (!token) {
+        setError('Not authenticated');
+        setRoomState('error');
+        return;
+      }
+
+      // Request session creation from backend
+      const resp = await fetch(`${apiUrl}/v1/voice/token`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify({
+          model: 'alia-v1-voice',
+          voice: voicePref === 'male' ? 'echo' : 'nova',
+        }),
+      });
+
+      if (!resp.ok) {
+        const body = await resp.json().catch(() => ({}));
+        const errMsg = body?.error?.message || body?.error || `Voice session failed (${resp.status})`;
+        setError(errMsg);
+        setRoomState('error');
+        return;
+      }
+
+      const { token: livekitToken, url, roomName, sessionId } = await resp.json();
+      sessionIdRef.current = sessionId;
+
+      // Create LiveKit room and connect
+      const room = new Room();
+      roomRef.current = room;
+
+      // Event: remote audio tracks (agent speaking) — must attach to play
+      room.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
+        if (track.kind === Track.Kind.Audio) {
+          track.attach();
+        }
+      });
+
+      room.on(RoomEvent.TrackUnsubscribed, (track) => {
+        track.detach();
+      });
+
+      // Event: data messages from agent
+      room.on(RoomEvent.DataReceived, (payload: Uint8Array) => {
+        handleDataMessage(payload);
+      });
+
+      // Event: disconnected
+      room.on(RoomEvent.Disconnected, (reason?: DisconnectReason) => {
+        setRoomState('disconnected');
+        setAgentState('idle');
+        setCohostActive(false);
+      });
+
+      // Connect to the room
+      await room.connect(url, livekitToken, { autoSubscribe: true });
+
+      // Enable microphone — audio goes to LiveKit, agent picks it up server-side
+      await room.localParticipant.setMicrophoneEnabled(true);
+
+      setRoomState('connected');
+      setAgentState('listening');
+
+    } catch (e: any) {
+      console.error('[useVoiceRoom] Connection error:', e);
+      setError(e.message || 'Failed to connect');
+      setRoomState('error');
+      cleanup();
+    }
+  }, [getToken, cleanup, handleDataMessage, apiUrl, voicePref]);
+
+  // ============== DISCONNECT ==============
+
+  const disconnect = useCallback(() => {
+    cleanup();
+    setRoomState('disconnected');
+    setAgentState('idle');
+    setError(null);
+    setIsMuted(false);
+    setMessages([]);
+    setCohostActive(false);
+    setCurrentSpeaker(null);
+    setRoundComplete(false);
+  }, [cleanup]);
+
+  // ============== MUTE ==============
+
+  const toggleMute = useCallback(() => {
+    const room = roomRef.current;
+    if (!room) return;
+    const next = !isMuted;
+    room.localParticipant.setMicrophoneEnabled(!next);
+    setIsMuted(next);
+  }, [isMuted]);
+
+  // ============== COHOST CONTROLS ==============
+
+  const sendClientData = useCallback((data: object) => {
+    const room = roomRef.current;
+    if (!room?.localParticipant) return;
+    const encoded = new TextEncoder().encode(JSON.stringify(data));
+    room.localParticipant.publishData(encoded, { reliable: true } as DataPublishOptions);
+  }, []);
+
+  const enableCohost = useCallback(() => {
+    sendClientData({ type: 'cohost.enable' });
+  }, [sendClientData]);
+
+  const disableCohost = useCallback(() => {
+    sendClientData({ type: 'cohost.disable' });
+  }, [sendClientData]);
+
+  const continueCohost = useCallback(() => {
+    setRoundComplete(false);
+    sendClientData({ type: 'cohost.continue' });
+  }, [sendClientData]);
+
+  // ============== CLEANUP ON UNMOUNT ==============
+
+  useEffect(() => cleanup, [cleanup]);
+
+  return {
+    room: roomRef.current,
+    roomState,
+    agentState,
+    isMuted,
+    error,
+    messages,
+    cohostActive,
+    currentSpeaker,
+    roundComplete,
+    connect,
+    disconnect,
+    toggleMute,
+    enableCohost,
+    disableCohost,
+    continueCohost,
+    isConnected: roomState === 'connected',
+  };
+}

package/src/index.ts

@@ -1,17 +1,51 @@
+/// <reference path="./optional-deps.d.ts" />
 // @alia.onl/sdk — Alia AI Chat SDK
-// Reusable bottom sheet chat component for any Oxy ecosystem app
+// Reusable chat UI, voice, and streaming components for Oxy apps
 
-// Main component
+// ── Main component ──
 export { AliaChatSheet } from './components/AliaChatSheet';
 export type { AliaChatSheetProps, AliaChatSheetRef } from './components/AliaChatSheet';
 
-// Hook (for custom UIs)
+// ── Chat UI components ──
+export { AliaFace } from './components/AliaFace';
+export type { AliaExpression, AliaFaceProps } from './components/AliaFace';
+export { ThinkingIndicator } from './components/ThinkingIndicator';
+export { AliaMarkdown } from './components/Markdown';
+export { Reasoning, ReasoningTrigger, ReasoningContent } from './components/Reasoning';
+
+// ── Voice components ──
+export { AudioWaveVisualizer } from './components/voice/AudioWaveVisualizer';
+export { VoiceOverlay } from './components/voice/VoiceOverlay';
+export { VoiceControls } from './components/voice/VoiceControls';
+
+// ── Chat hook ──
 export { useAliaChat } from './hooks/useAliaChat';
 export type { UseAliaChatOptions, UseAliaChatReturn } from './hooks/useAliaChat';
 
-// Types
-export type { ChatMessage, ToolInvocation, AliaChatSuggestion } from './types';
+// ── Voice hooks ──
+export { useVoiceRoom } from './hooks/useVoiceRoom';
+export { useAudioLevelMonitor } from './hooks/useAudioLevelMonitor';
+export { useAudioLevels } from './hooks/useAudioLevels';
+export { useTTS } from './hooks/useTTS';
+export type { UseTTSOptions } from './hooks/useTTS';
+export { useSpeechToText } from './hooks/useSpeechToText';
+export type { UseSTTOptions } from './hooks/useSpeechToText';
+export { useSoundEffects, useVoiceSoundEffects } from './hooks/useSoundEffects';
+export type { SoundName, SoundSources } from './hooks/useSoundEffects';
+
+// ── Types ──
+export type {
+  ChatMessage,
+  ToolInvocation,
+  AliaChatSuggestion,
+  ResearchProgress,
+  PendingPlan,
+  RoomState,
+  AgentState,
+  VoiceMessage,
+  VoiceToolInvocation,
+} from './types';
 
-// Theme
+// ── Theme ──
 export { useAliaColors } from './theme';
 export type { AliaColors } from './theme';