@alia.onl/sdk 1.0.0 → 3.0.0

package/src/hooks/useDocumentPicker.ts

@@ -0,0 +1,37 @@
+import * as DocumentPicker from 'expo-document-picker';
+
+export type DocumentPickerResult = {
+  uri: string;
+  name: string;
+  size: number;
+  mimeType: string;
+};
+
+type UseDocumentPickerResult = {
+  pickDocument: () => Promise<DocumentPickerResult[] | undefined>;
+};
+
+export function useDocumentPicker(): UseDocumentPickerResult {
+  const pickDocument = async (): Promise<DocumentPickerResult[] | undefined> => {
+    try {
+      const result = await DocumentPicker.getDocumentAsync({
+        type: '*/*',
+        multiple: true,
+        copyToCacheDirectory: true,
+      });
+
+      if (result.canceled === false && result.assets && result.assets.length > 0) {
+        return result.assets.map((asset: any) => ({
+          uri: asset.uri,
+          name: asset.name,
+          size: asset.size || 0,
+          mimeType: asset.mimeType || 'application/octet-stream',
+        }));
+      }
+    } catch (error) {
+      console.warn('Failed to pick document:', error);
+    }
+  };
+
+  return { pickDocument };
+}

package/src/hooks/useImagePicker.ts

@@ -0,0 +1,38 @@
+import * as ImagePicker from 'expo-image-picker';
+
+export type ImagePickerAsset = {
+  uri: string;
+  name: string;
+  size: number;
+  mimeType: string;
+};
+
+type ImagePickerResult = {
+  pickImage: () => Promise<ImagePickerAsset[] | undefined>;
+};
+
+export function useImagePicker(): ImagePickerResult {
+  const pickImage = async (): Promise<ImagePickerAsset[] | undefined> => {
+    try {
+      const result = await ImagePicker.launchImageLibraryAsync({
+        allowsMultipleSelection: true,
+        mediaTypes: ImagePicker.MediaTypeOptions.Images,
+        aspect: [4, 3],
+        quality: 1,
+      });
+
+      if (!result.canceled && result.assets.length > 0) {
+        return result.assets.map((asset: any) => ({
+          uri: asset.uri,
+          name: asset.fileName || `image-${Date.now()}.jpg`,
+          size: asset.fileSize || 0,
+          mimeType: asset.mimeType || 'image/jpeg',
+        }));
+      }
+    } catch (error) {
+      console.warn('Failed to pick image:', error);
+    }
+  };
+
+  return { pickImage };
+}

package/src/hooks/useSoundEffects.ts

@@ -0,0 +1,128 @@
+/**
+ * Sound effects hook for voice mode.
+ * Plays short audio cues when the AI is thinking, using tools,
+ * or when voice connection state changes.
+ *
+ * Sounds are provided via options instead of hardcoded require() calls,
+ * making this hook portable across different bundlers and apps.
+ */
+
+import { useEffect, useRef, useCallback } from 'react';
+import type { AgentState } from '../types';
+
+export type SoundName = 'thinking' | 'toolCall' | 'voiceConnect' | 'voiceDisconnect';
+
+export interface SoundSources {
+  thinking?: any;
+  toolCall?: any;
+  voiceConnect?: any;
+  voiceDisconnect?: any;
+}
+
+const SOUND_NAMES: SoundName[] = ['thinking', 'toolCall', 'voiceConnect', 'voiceDisconnect'];
+
+export function useSoundEffects(enabled: boolean = false, sounds: SoundSources = {}) {
+  const playersRef = useRef<Map<SoundName, any>>(new Map());
+  const loadedRef = useRef(false);
+
+  useEffect(() => {
+    if (!enabled || loadedRef.current) return;
+
+    // Only load sounds that were actually provided
+    const hasAnySounds = SOUND_NAMES.some((name) => sounds[name] != null);
+    if (!hasAnySounds) return;
+
+    let mounted = true;
+
+    (async () => {
+      try {
+        const { createAudioPlayer } = await import('expo-audio');
+        if (!mounted) return;
+
+        for (const name of SOUND_NAMES) {
+          const source = sounds[name];
+          if (source == null) continue;
+          try {
+            const player = createAudioPlayer(source);
+            playersRef.current.set(name, player);
+          } catch {
+            // Individual sound failed to load — skip it
+          }
+        }
+        loadedRef.current = true;
+      } catch {
+        // expo-audio not available — degrade silently
+      }
+    })();
+
+    return () => {
+      mounted = false;
+      for (const player of playersRef.current.values()) {
+        try { player.release(); } catch {}
+      }
+      playersRef.current.clear();
+      loadedRef.current = false;
+    };
+  }, [enabled, sounds]);
+
+  const play = useCallback((sound: SoundName) => {
+    if (!enabled) return;
+    const player = playersRef.current.get(sound);
+    if (player) {
+      try {
+        player.seekTo(0);
+        player.play();
+      } catch {
+        // Playback failed — ignore
+      }
+    }
+  }, [enabled]);
+
+  return { play };
+}
+
+/**
+ * Hook that automatically triggers sound effects based on voice state changes.
+ * Wire this into the conversation page alongside useVoiceMode.
+ */
+export function useVoiceSoundEffects({
+  isVoiceActive,
+  agentState,
+  isConnected,
+  sounds = {},
+}: {
+  isVoiceActive: boolean;
+  agentState: AgentState;
+  isConnected: boolean;
+  sounds?: SoundSources;
+}) {
+  const { play } = useSoundEffects(isVoiceActive, sounds);
+  const prevAgentStateRef = useRef<AgentState>('idle');
+  const prevConnectedRef = useRef(false);
+
+  // Play sound on agent state transitions
+  useEffect(() => {
+    if (!isVoiceActive) return;
+
+    const prev = prevAgentStateRef.current;
+    prevAgentStateRef.current = agentState;
+
+    if (agentState === 'thinking' && prev !== 'thinking') {
+      play('thinking');
+    }
+  }, [agentState, isVoiceActive, play]);
+
+  // Play sound on connection state changes
+  useEffect(() => {
+    if (!isVoiceActive) return;
+
+    const wasConnected = prevConnectedRef.current;
+    prevConnectedRef.current = isConnected;
+
+    if (isConnected && !wasConnected) {
+      play('voiceConnect');
+    } else if (!isConnected && wasConnected) {
+      play('voiceDisconnect');
+    }
+  }, [isConnected, isVoiceActive, play]);
+}

package/src/hooks/useSpeechToText.ts

@@ -0,0 +1,182 @@
+import { useState, useCallback, useRef, useEffect } from 'react';
+import {
+  useAudioRecorder,
+  useAudioRecorderState,
+  RecordingPresets,
+  requestRecordingPermissionsAsync,
+  setAudioModeAsync,
+} from 'expo-audio';
+import { useOxy } from '@oxyhq/services';
+import { create } from 'zustand';
+
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+
+// ============== OPTIONS ==============
+
+export interface UseSTTOptions {
+  apiUrl?: string;
+  accessToken?: string;
+}
+
+// ============== INLINE STT STORE ==============
+
+interface STTStoreState {
+  isRecording: boolean;
+  metering: number;
+  setRecording: (v: boolean) => void;
+  setMetering: (v: number) => void;
+}
+
+export const useSTTStore = create<STTStoreState>((set) => ({
+  isRecording: false,
+  metering: 0,
+  setRecording: (isRecording) => set({ isRecording }),
+  setMetering: (metering) => set({ metering }),
+}));
+
+// ============== HOOK ==============
+
+type STTState = 'idle' | 'recording' | 'transcribing';
+
+const METERING_PRESET = { ...RecordingPresets.HIGH_QUALITY, isMeteringEnabled: true };
+
+export function useSpeechToText(options: UseSTTOptions = {}) {
+  const apiUrl = options.apiUrl || API_URL;
+
+  const [state, setState] = useState<STTState>('idle');
+  const [error, setError] = useState<string | null>(null);
+  const isRecordingRef = useRef(false);
+  const { oxyServices } = useOxy();
+
+  const audioRecorder = useAudioRecorder(METERING_PRESET);
+  const recorderState = useAudioRecorderState(audioRecorder, 100);
+
+  const sttStore = useSTTStore;
+
+  // ============== AUTH ==============
+
+  const getToken = useCallback((): string | null => {
+    if (options.accessToken) return options.accessToken;
+    return oxyServices.httpService.getAccessToken();
+  }, [options.accessToken, oxyServices]);
+
+  // Push metering to store while recording
+  useEffect(() => {
+    if (state === 'recording' && recorderState.metering != null) {
+      // dBFS: -160 (silence) to 0 (max). Normalize using -60 as practical floor.
+      const normalized = Math.min(1, Math.max(0, (recorderState.metering + 60) / 60));
+      sttStore.getState().setMetering(normalized);
+    }
+  }, [state, recorderState.metering]);
+
+  // Sync recording state to store
+  useEffect(() => {
+    sttStore.getState().setRecording(state === 'recording');
+  }, [state]);
+
+  const startRecording = useCallback(async () => {
+    try {
+      setError(null);
+
+      const permStatus = await requestRecordingPermissionsAsync();
+      if (!permStatus.granted) {
+        setError('Microphone permission required');
+        return;
+      }
+
+      setState('recording');
+
+      await setAudioModeAsync({
+        allowsRecording: true,
+        playsInSilentMode: true,
+      });
+
+      await audioRecorder.prepareToRecordAsync();
+      audioRecorder.record();
+      isRecordingRef.current = true;
+    } catch (e: any) {
+      console.error('[STT] Recording error:', e);
+      setError('Failed to start recording');
+      setState('idle');
+      isRecordingRef.current = false;
+    }
+  }, [audioRecorder]);
+
+  const stopAndTranscribe = useCallback(async (): Promise<string | null> => {
+    if (!isRecordingRef.current) return null;
+
+    try {
+      setState('transcribing');
+      isRecordingRef.current = false;
+      await audioRecorder.stop();
+      const uri = audioRecorder.uri;
+
+      if (!uri) {
+        setState('idle');
+        return null;
+      }
+
+      // Read audio file as base64
+      const response = await fetch(uri);
+      const blob = await response.blob();
+      // Detect actual MIME type (web records webm, native records m4a)
+      const detectedFormat = blob.type?.split(';')[0] || 'audio/m4a';
+      const base64 = await new Promise<string>((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => {
+          const result = reader.result as string;
+          resolve(result.split(',')[1]);
+        };
+        reader.onerror = reject;
+        reader.readAsDataURL(blob);
+      });
+
+      // Send to transcription API
+      const token = getToken();
+      const transcribeResponse = await fetch(`${apiUrl}/v1/voice/transcribe`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify({
+          audio: base64,
+          format: detectedFormat,
+        }),
+      });
+
+      if (!transcribeResponse.ok) {
+        const errorData = await transcribeResponse.json().catch(() => ({}));
+        throw new Error(errorData.error || 'Transcription failed');
+      }
+
+      const result = await transcribeResponse.json() as { text: string };
+      setState('idle');
+      return result.text || null;
+    } catch (e: any) {
+      console.error('[STT] Transcription error:', e);
+      setError(e.message || 'Transcription failed');
+      setState('idle');
+      return null;
+    }
+  }, [audioRecorder, getToken, apiUrl]);
+
+  const cancel = useCallback(() => {
+    if (isRecordingRef.current) {
+      isRecordingRef.current = false;
+      Promise.resolve(audioRecorder.stop()).catch(() => {});
+    }
+    setState('idle');
+    setError(null);
+  }, [audioRecorder]);
+
+  return {
+    state,
+    error,
+    startRecording,
+    stopAndTranscribe,
+    cancel,
+    isRecording: state === 'recording',
+    isTranscribing: state === 'transcribing',
+  };
+}

package/src/hooks/useTTS.ts

@@ -0,0 +1,233 @@
+import { useCallback, useEffect, useRef } from 'react';
+import {
+  useSharedValue,
+  withRepeat,
+  withSequence,
+  withTiming,
+  cancelAnimation,
+  Easing,
+} from 'react-native-reanimated';
+import { useOxy } from '@oxyhq/services';
+import { create } from 'zustand';
+
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+
+// ============== OPTIONS ==============
+
+export interface UseTTSOptions {
+  apiUrl?: string;
+  accessToken?: string;
+  voice?: 'male' | 'female';
+  tone?: 'brief' | 'chill' | 'default';
+}
+
+// ============== INLINE TTS STORE ==============
+
+type PlaybackState = 'idle' | 'loading' | 'playing' | 'paused' | 'error';
+
+interface TTSStore {
+  activeMessageId: string | null;
+  playbackState: PlaybackState;
+  error: string | null;
+  setActiveMessage: (id: string | null) => void;
+  setPlaybackState: (state: PlaybackState) => void;
+  setError: (error: string | null) => void;
+  reset: () => void;
+}
+
+const useTTSStore = create<TTSStore>((set) => ({
+  activeMessageId: null,
+  playbackState: 'idle',
+  error: null,
+  setActiveMessage: (id) => set({ activeMessageId: id }),
+  setPlaybackState: (playbackState) => set({ playbackState }),
+  setError: (error) => set({ error, playbackState: error ? 'error' : 'idle' }),
+  reset: () => set({ activeMessageId: null, playbackState: 'idle', error: null }),
+}));
+
+// ============== HOOK ==============
+
+export function useTTS(options: UseTTSOptions = {}) {
+  const apiUrl = options.apiUrl || API_URL;
+  const voicePref = options.voice ?? 'female';
+  const tonePref = options.tone ?? 'default';
+
+  const { oxyServices } = useOxy();
+  const {
+    activeMessageId,
+    playbackState,
+    error,
+    setActiveMessage,
+    setPlaybackState,
+    setError,
+    reset,
+  } = useTTSStore();
+
+  const playerRef = useRef<any>(null);
+
+  // Simulated wave amplitude for visualization
+  const ttsWaveAmplitude = useSharedValue(0);
+
+  // ============== AUTH ==============
+
+  const getToken = useCallback((): string | null => {
+    if (options.accessToken) return options.accessToken;
+    return oxyServices.httpService.getAccessToken();
+  }, [options.accessToken, oxyServices]);
+
+  // Animate wave when playing
+  useEffect(() => {
+    if (playbackState === 'playing') {
+      ttsWaveAmplitude.value = withRepeat(
+        withSequence(
+          withTiming(0.5, { duration: 800, easing: Easing.inOut(Easing.sin) }),
+          withTiming(0.2, { duration: 600, easing: Easing.inOut(Easing.sin) }),
+          withTiming(0.6, { duration: 700, easing: Easing.inOut(Easing.sin) }),
+          withTiming(0.15, { duration: 500, easing: Easing.inOut(Easing.sin) }),
+        ),
+        -1,
+      );
+    } else {
+      cancelAnimation(ttsWaveAmplitude);
+      ttsWaveAmplitude.value = withTiming(0, { duration: 300 });
+    }
+  }, [playbackState]);
+
+  const getTTSVoice = useCallback(() => {
+    return voicePref === 'male' ? 'echo' : 'nova';
+  }, [voicePref]);
+
+  const getTTSSpeed = useCallback(() => {
+    if (tonePref === 'brief') return 1.15;
+    if (tonePref === 'chill') return 0.9;
+    return 1.0;
+  }, [tonePref]);
+
+  const releasePlayer = useCallback(() => {
+    try {
+      playerRef.current?.remove();
+    } catch {}
+    playerRef.current = null;
+  }, []);
+
+  const stop = useCallback(() => {
+    releasePlayer();
+    reset();
+  }, [reset, releasePlayer]);
+
+  const playFromUrl = useCallback((audioUrl: string, _messageId: string) => {
+    releasePlayer();
+
+    (async () => {
+      try {
+        const { createAudioPlayer } = await import('expo-audio');
+        const player = createAudioPlayer({ uri: audioUrl });
+        playerRef.current = player;
+
+        player.addListener('playbackStatusUpdate', (status: any) => {
+          if (status.didJustFinish) {
+            releasePlayer();
+            reset();
+          }
+        });
+
+        player.play();
+        setPlaybackState('playing');
+      } catch {
+        setError('Audio playback not available');
+      }
+    })();
+  }, [releasePlayer, reset, setPlaybackState, setError]);
+
+  const readAloud = useCallback(async (
+    messageId: string,
+    text: string,
+    conversationId?: string,
+    audioUrl?: string,
+  ) => {
+    // If same message is playing, toggle pause/play
+    if (activeMessageId === messageId) {
+      if (playbackState === 'playing') {
+        playerRef.current?.pause();
+        setPlaybackState('paused');
+        return;
+      }
+      if (playbackState === 'paused') {
+        playerRef.current?.play();
+        setPlaybackState('playing');
+        return;
+      }
+    }
+
+    // Stop any current playback
+    if (activeMessageId) {
+      stop();
+    }
+
+    try {
+      setActiveMessage(messageId);
+      setPlaybackState('loading');
+
+      // If cached audioUrl exists, play directly
+      if (audioUrl) {
+        playFromUrl(audioUrl, messageId);
+        return;
+      }
+
+      // Call backend TTS API
+      const token = getToken();
+      if (!token) {
+        throw new Error('Not authenticated');
+      }
+
+      const response = await fetch(`${apiUrl}/v1/audio/speech`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify({
+          model: 'alia-v1-voice',
+          input: text,
+          voice: getTTSVoice(),
+          speed: getTTSSpeed(),
+          conversationId,
+          messageId,
+        }),
+      });
+
+      if (!response.ok) {
+        if (response.status === 504) {
+          throw new Error('Request timed out — please try again');
+        }
+        const errData = await response.json().catch(() => ({}));
+        throw new Error(errData.error?.message || errData.error || 'TTS failed');
+      }
+
+      const data = await response.json();
+      playFromUrl(data.audioUrl, messageId);
+    } catch (e: any) {
+      console.error('[TTS] Error:', e);
+      setError(e.message || 'Failed to read aloud');
+    }
+  }, [activeMessageId, playbackState, getToken, apiUrl, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      releasePlayer();
+    };
+  }, []);
+
+  return {
+    readAloud,
+    stop,
+    activeMessageId,
+    playbackState,
+    error,
+    ttsWaveAmplitude,
+    isPlaying: playbackState === 'playing',
+    isPaused: playbackState === 'paused',
+    isLoading: playbackState === 'loading',
+  };
+}