@alia.onl/sdk 7.2.9 → 10.0.0

package/src/hooks/useSpeechToText.ts

@@ -1,23 +1,24 @@
 import { useState, useCallback, useRef, useEffect } from 'react';
-import { errorMessage } from '../lib/utils';
-import {
-  useAudioRecorder,
-  useAudioRecorderState,
-  RecordingPresets,
-  getRecordingPermissionsAsync,
-  setAudioModeAsync,
-} from 'expo-audio';
-import { useOxy } from '@oxy.so/services';
 import { create } from 'zustand';
-import { levelFromDbfs } from '../lib/audio-level';
-
-const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+import {
+  chooseSpeechRecognizer,
+  isSpeechRecognitionAvailable,
+  requestSpeechRecognitionPermission,
+  startSpeechRecognition,
+} from '../lib/speech-recognition';
+import type { SpeechRecognitionSession } from '../lib/speech-recognition-types';
+import type { SpeechRecognizerChoice } from '../lib/speech-recognition-types';
+import { speechFailureCode, VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
+import { defaultSpeechLanguage } from '../lib/speech-language';
 
 // ============== OPTIONS ==============
 
 export interface UseSTTOptions {
-  apiUrl?: string;
-  accessToken?: string;
+  /**
+   * Language to recognize, as a BCP-47 tag (`es-ES`). Defaults to the
+   * platform's own language. The Alia app passes its UI locale.
+   */
+  lang?: string;
 }
 
 // ============== INLINE STT STORE ==============
@@ -40,169 +41,217 @@ export const useSTTStore = create<STTStoreState>((set) => ({
 
 type STTState = 'idle' | 'recording' | 'transcribing';
 
-const METERING_PRESET = { ...RecordingPresets.HIGH_QUALITY, isMeteringEnabled: true };
+/** How long `stopAndTranscribe` waits for the engine's final words. */
+const FINAL_RESULT_TIMEOUT_MS = 2_000;
 
+/**
+ * How many times one dictation may quietly re-open the recognizer.
+ *
+ * Browsers end a continuous session on their own — Chrome after a stretch of
+ * silence or about a minute of audio — and the person is still looking at a
+ * dictation bar that says it is listening. Re-opening keeps what was already
+ * heard and carries on. The cap is what stops an engine that ends the instant
+ * it starts from spinning.
+ */
+const MAX_SILENT_RESTARTS = 20;
+
+/**
+ * Dictation, recognized on the device.
+ *
+ * The public shape is the one the recorder-and-upload version had —
+ * `startRecording`, `stopAndTranscribe`, `cancel`, the three states and
+ * `useSTTStore`'s metering — so the composer and the dictation bar did not
+ * change. What changed underneath: words arrive while the person speaks
+ * (`speech-recognition.ts` on web, `.native.ts` on iOS/Android), so
+ * `transcribing` is now the moment between "stop" and the engine's final
+ * result rather than an upload, and there is no transcription endpoint.
+ */
 export function useSpeechToText(options: UseSTTOptions = {}) {
-  const apiUrl = options.apiUrl || API_URL;
+  const lang = options.lang ?? defaultSpeechLanguage();
 
   const [state, setState] = useState<STTState>('idle');
-  const [error, setError] = useState<string | null>(null);
-  const isRecordingRef = useRef(false);
-  const { oxyServices } = useOxy();
-
-  const audioRecorder = useAudioRecorder(METERING_PRESET);
-  const recorderState = useAudioRecorderState(audioRecorder, 100);
+  const [errorCode, setErrorCode] = useState<VoiceErrorCode | null>(null);
+  const [isSupported] = useState(isSpeechRecognitionAvailable);
 
-  const sttStore = useSTTStore;
+  const stateRef = useRef<STTState>('idle');
+  const sessionRef = useRef<SpeechRecognitionSession | null>(null);
+  /** Text from sessions that already ended during this dictation. */
+  const earlierTextRef = useRef('');
+  /** Text from the live session. */
+  const currentTextRef = useRef('');
+  const restartsRef = useRef(0);
+  /** The recognizer and tag this dictation uses, chosen once when it starts. */
+  const recognizerRef = useRef<{ lang: string; service?: string; silent: boolean }>({ lang, silent: false });
+  const endWaitersRef = useRef<Array<() => void>>([]);
+  const lastMeteringRef = useRef(0);
+  const mountedRef = useRef(true);
 
-  // ============== AUTH ==============
+  /** What went wrong, as a code; `null` clears it. */
+  const report = useCallback((code: VoiceErrorCode | null) => {
+    if (mountedRef.current) setErrorCode(code);
+  }, []);
 
-  const getToken = useCallback((): string | null => {
-    if (options.accessToken) return options.accessToken;
-    return oxyServices.httpService.getAccessToken();
-  }, [options.accessToken, oxyServices]);
-
-  // Push metering to store while recording (with epsilon guard to avoid thrashing subscribers)
-  const lastMeteringRef = useRef(0);
-  useEffect(() => {
-    if (state === 'recording') {
-      let target: number;
-      if (recorderState.metering != null) {
-        // The same curve read-aloud playback uses, so one background answers
-        // both the same way — see `lib/audio-level.ts`.
-        target = levelFromDbfs(recorderState.metering);
-      } else {
-        // Metering unavailable — simulate gentle activity
-        target = 0.12 + Math.random() * 0.18;
-      }
-      if (Math.abs(target - lastMeteringRef.current) >= 0.02) {
-        lastMeteringRef.current = target;
-        sttStore.getState().setMetering(target);
-      }
-    }
-  }, [state, recorderState.metering, recorderState.durationMillis]);
+  const setPhase = useCallback((next: STTState) => {
+    stateRef.current = next;
+    if (mountedRef.current) setState(next);
+  }, []);
 
   // Sync recording state to store + reset metering on stop
   useEffect(() => {
-    sttStore.getState().setRecording(state === 'recording');
+    useSTTStore.getState().setRecording(state === 'recording');
     if (state !== 'recording') {
       lastMeteringRef.current = 0;
-      sttStore.getState().setMetering(0);
+      useSTTStore.getState().setMetering(0);
     }
   }, [state]);
 
-  const startRecording = useCallback(async () => {
-    try {
-      setError(null);
-
-      // getRecordingPermissionsAsync consults the Permissions API first (and
-      // only falls back to a real getUserMedia prompt when undetermined), so an
-      // already-granted permission never gets misreported as denied when the
-      // device itself is missing/busy — that failure surfaces from
-      // prepareToRecordAsync below with an accurate message.
-      const permStatus = await getRecordingPermissionsAsync();
-      if (!permStatus.granted) {
-        setError('Microphone permission required');
-        return;
-      }
+  const fullText = (): string =>
+    `${earlierTextRef.current} ${currentTextRef.current}`.replace(/\s+/g, ' ').trim();
 
-      setState('recording');
+  const openSession = useCallback(() => {
+    currentTextRef.current = '';
+    let failed = false;
+    const recognizer = recognizerRef.current;
+    sessionRef.current = startSpeechRecognition(
+      { lang: recognizer.lang, service: recognizer.service },
+      {
+        onResult: ({ transcript }) => {
+          currentTextRef.current = transcript;
+        },
+        onLevel: (level) => {
+          if (stateRef.current !== 'recording') return;
+          // Epsilon guard: quiet should not thrash the store's subscribers.
+          if (Math.abs(level - lastMeteringRef.current) < 0.02) return;
+          lastMeteringRef.current = level;
+          useSTTStore.getState().setMetering(level);
+        },
+        onError: (failure) => {
+          const code = speechFailureCode(failure);
+          if (code === null) return;
+          failed = true;
+          // Every recognizer on the device named no language, and the
+          // default refused this one: it is the device that cannot recognize
+          // speech, not the language that is wrong.
+          report(code === 'speech-language' && recognizer.silent ? 'speech-unsupported' : code);
+        },
+        onEnd: () => {
+          sessionRef.current = null;
+          earlierTextRef.current = fullText();
+          currentTextRef.current = '';
+          const waiters = endWaitersRef.current;
+          endWaitersRef.current = [];
+          for (const resolve of waiters) resolve();
+          if (waiters.length > 0) return;
 
-      await setAudioModeAsync({
-        allowsRecording: true,
-        playsInSilentMode: true,
-      });
+          // The engine ended on its own while the person is still dictating.
+          if (stateRef.current !== 'recording') return;
+          if (failed || restartsRef.current >= MAX_SILENT_RESTARTS) {
+            setPhase('idle');
+            return;
+          }
+          restartsRef.current += 1;
+          openSession();
+        },
+      },
+    );
+  }, [report, setPhase]);
 
-      await audioRecorder.prepareToRecordAsync();
-      audioRecorder.record();
-      isRecordingRef.current = true;
-    } catch (e: unknown) {
-      const name = e instanceof Error ? e.name : '';
-      if (name === 'NotFoundError' || name === 'OverconstrainedError') {
-        setError('No microphone found — check your input devices');
-      } else if (name === 'NotAllowedError') {
-        setError('Microphone permission required');
-      } else if (name === 'NotReadableError') {
-        setError('Microphone is in use by another application');
-      } else {
-        setError('Failed to start recording');
-      }
-      setState('idle');
-      isRecordingRef.current = false;
-    }
-  }, [audioRecorder]);
+  const startRecording = useCallback(async () => {
+    if (stateRef.current !== 'idle') return;
+    report(null);
 
-  const stopAndTranscribe = useCallback(async (): Promise<string | null> => {
-    if (!isRecordingRef.current) return null;
+    if (!isSpeechRecognitionAvailable()) {
+      report('speech-unsupported');
+      return;
+    }
 
-    try {
-      setState('transcribing');
-      isRecordingRef.current = false;
-      await audioRecorder.stop();
-      const uri = audioRecorder.uri;
+    // Claim the state before the first await so a double tap cannot open two.
+    setPhase('recording');
+    const refusal = await requestSpeechRecognitionPermission();
+    if ((stateRef.current as STTState) !== 'recording') return; // cancelled while asking
+    if (refusal !== null) {
+      report(speechFailureCode(refusal));
+      setPhase('idle');
+      return;
+    }
 
-      if (!uri) {
-        setState('idle');
-        return null;
-      }
+    const choice: SpeechRecognizerChoice = await chooseSpeechRecognizer(lang);
+    if ((stateRef.current as STTState) !== 'recording') return; // cancelled while choosing
+    if ('failure' in choice) {
+      report(speechFailureCode(choice.failure));
+      setPhase('idle');
+      return;
+    }
+    recognizerRef.current = choice;
 
-      // Read audio file as base64
-      const response = await fetch(uri);
-      const blob = await response.blob();
-      // Detect actual MIME type (web records webm, native records m4a)
-      const detectedFormat = blob.type?.split(';')[0] || 'audio/m4a';
-      const base64 = await new Promise<string>((resolve, reject) => {
-        const reader = new FileReader();
-        reader.onload = () => {
-          const result = reader.result as string;
-          resolve(result.split(',')[1]);
-        };
-        reader.onerror = reject;
-        reader.readAsDataURL(blob);
-      });
+    earlierTextRef.current = '';
+    currentTextRef.current = '';
+    restartsRef.current = 0;
+    openSession();
+  }, [lang, openSession, report, setPhase]);
 
-      // Send to transcription API
-      const token = getToken();
-      const transcribeResponse = await fetch(`${apiUrl}/v1/voice/transcribe`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${token}`,
-        },
-        body: JSON.stringify({
-          audio: base64,
-          format: detectedFormat,
-        }),
+  const waitForEnd = useCallback((): Promise<void> => {
+    if (sessionRef.current === null) return Promise.resolve();
+    return new Promise<void>((resolve) => {
+      const timer = setTimeout(() => {
+        sessionRef.current?.abort();
+        resolve();
+      }, FINAL_RESULT_TIMEOUT_MS);
+      endWaitersRef.current.push(() => {
+        clearTimeout(timer);
+        resolve();
       });
+    });
+  }, []);
 
-      if (!transcribeResponse.ok) {
-        const errorData = await transcribeResponse.json().catch(() => ({}));
-        throw new Error(errorData.error || 'Transcription failed');
-      }
-
-      const result = await transcribeResponse.json() as { text: string };
-      setState('idle');
-      return result.text || null;
-    } catch (e: unknown) {
-      console.error('[STT] Transcription error:', e);
-      setError(errorMessage(e, 'Transcription failed'));
-      setState('idle');
-      return null;
-    }
-  }, [audioRecorder, getToken, apiUrl]);
+  const stopAndTranscribe = useCallback(async (): Promise<string | null> => {
+    if (stateRef.current !== 'recording') return null;
+    setPhase('transcribing');
+    const ended = waitForEnd();
+    sessionRef.current?.stop();
+    await ended;
+    const text = fullText();
+    earlierTextRef.current = '';
+    currentTextRef.current = '';
+    setPhase('idle');
+    return text === '' ? null : text;
+  }, [setPhase, waitForEnd]);
 
   const cancel = useCallback(() => {
-    if (isRecordingRef.current) {
-      isRecordingRef.current = false;
-      Promise.resolve(audioRecorder.stop()).catch(() => {});
-    }
-    setState('idle');
-    setError(null);
-  }, [audioRecorder]);
+    const session = sessionRef.current;
+    setPhase('idle');
+    earlierTextRef.current = '';
+    currentTextRef.current = '';
+    session?.abort();
+    report(null);
+  }, [report, setPhase]);
+
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      const wasListening = stateRef.current !== 'idle';
+      stateRef.current = 'idle';
+      sessionRef.current?.abort();
+      // The effect that mirrors the phase into the store will not run again,
+      // and the store outlives this hook: a composer unmounted mid-dictation —
+      // the call controls replacing it — would otherwise leave the ambient
+      // field and the dictation bar reading "recording" for the rest of the call.
+      if (wasListening) {
+        useSTTStore.getState().setRecording(false);
+        useSTTStore.getState().setMetering(0);
+      }
+    };
+  }, []);
 
   return {
     state,
-    error,
+    /** What went wrong, in English. */
+    error: errorCode === null ? null : VOICE_ERROR_MESSAGES[errorCode],
+    /** The same, as a code an app can translate. */
+    errorCode,
+    /** False where the platform cannot recognize speech at all (Firefox). */
+    isSupported,
     startRecording,
     stopAndTranscribe,
     cancel,

package/src/hooks/useTTS.ts

@@ -5,7 +5,7 @@ import { useOxy } from '@oxy.so/services';
 import { errorMessage } from '../lib/utils';
 import { create } from 'zustand';
 import { createAudioLevelMeter } from '../lib/audio-level';
-import { PREFERRED_VOICE_MODEL_ID } from '../lib/config';
+import { requestSpeechClip } from '../lib/speech-synthesis';
 
 const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
 
@@ -17,12 +17,8 @@ export interface UseTTSOptions {
   voice?: 'male' | 'female';
   tone?: 'brief' | 'chill' | 'default';
   /**
-   * Speech model. Defaults to the build's `PREFERRED_VOICE_MODEL_ID`.
-   *
-   * Not checked against `GET /catalogue`: that surface describes what a chat
-   * picker may offer and its resolver filters to `chat_visible` entries, so
-   * passing a voice identifier through it would substitute a chat model that
-   * cannot speak. See `lib/config.ts`.
+   * Speech model. Omitted, the request carries no `model` and the server's
+   * own speech model answers.
    */
   model?: string;
 }
@@ -89,7 +85,7 @@ useTTSStore.subscribe((state, previous) => {
 export function useTTS(options: UseTTSOptions = {}) {
   const apiUrl = options.apiUrl || API_URL;
   const voicePref = options.voice ?? 'female';
-  const voiceModel = options.model ?? PREFERRED_VOICE_MODEL_ID;
+  const voiceModel = options.model;
   const tonePref = options.tone ?? 'default';
 
   const { oxyServices } = useOxy();
@@ -105,16 +101,17 @@ export function useTTS(options: UseTTSOptions = {}) {
   const reset = useTTSStore((s) => s.reset);
 
   const playerRef = useRef<AudioPlayer | null>(null);
+  const requestRef = useRef<AbortController | null>(null);
 
   // ============== AUTH ==============
 
   const getToken = useCallback((): string | null => {
     if (options.accessToken) return options.accessToken;
-    return oxyServices.httpService.getAccessToken();
+    return oxyServices.session.accessToken;
   }, [options.accessToken, oxyServices]);
 
   const getTTSVoice = useCallback(() => {
-    return voicePref === 'male' ? 'echo' : 'nova';
+    return voicePref === 'male' ? 'male' : 'female';
   }, [voicePref]);
 
   const getTTSSpeed = useCallback(() => {
@@ -133,6 +130,8 @@ export function useTTS(options: UseTTSOptions = {}) {
   }, []);
 
   const stop = useCallback(() => {
+    requestRef.current?.abort();
+    requestRef.current = null;
     releasePlayer();
     reset();
   }, [reset, releasePlayer]);
@@ -150,6 +149,7 @@ export function useTTS(options: UseTTSOptions = {}) {
   const playFromUrl = useCallback((
     audioUrl: string,
     _messageId: string,
+    signal: AbortSignal,
     onUnplayable?: () => void,
   ) => {
     releasePlayer();
@@ -157,6 +157,7 @@ export function useTTS(options: UseTTSOptions = {}) {
     (async () => {
       try {
         const { createAudioPlayer } = await import('expo-audio');
+        if (signal.aborted) return;
         /**
          * `crossOrigin` is web-only and ignored elsewhere, and it is what makes
          * the waveform readable there: a browser will not let an `AnalyserNode`
@@ -199,6 +200,7 @@ export function useTTS(options: UseTTSOptions = {}) {
         });
 
         player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
+          if (signal.aborted || playerRef.current !== player) return;
           if (status.error) {
             releasePlayer();
             if (onUnplayable) {
@@ -217,6 +219,7 @@ export function useTTS(options: UseTTSOptions = {}) {
         player.play();
         setPlaybackState('playing');
       } catch {
+        if (signal.aborted) return;
         if (onUnplayable) {
           onUnplayable();
           return;
@@ -256,6 +259,8 @@ export function useTTS(options: UseTTSOptions = {}) {
       stop();
     }
 
+    const controller = new AbortController();
+    requestRef.current = controller;
     try {
       setActiveMessage(messageId);
       setPlaybackState('loading');
@@ -274,32 +279,18 @@ export function useTTS(options: UseTTSOptions = {}) {
           throw new Error('Not authenticated');
         }
 
-        const response = await fetch(`${apiUrl}/v1/audio/speech`, {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${token}`,
-          },
-          body: JSON.stringify({
-            model: voiceModel,
-            input: text,
-            voice: getTTSVoice(),
-            speed: getTTSSpeed(),
-            conversationId,
-            messageId,
-          }),
+        const audioUrl = await requestSpeechClip({
+          apiUrl,
+          token,
+          model: voiceModel,
+          input: text,
+          voice: getTTSVoice(),
+          speed: getTTSSpeed(),
+          conversationId,
+          messageId,
+          signal: controller.signal,
         });
-
-        if (!response.ok) {
-          if (response.status === 504) {
-            throw new Error('Request timed out — please try again');
-          }
-          const errData = await response.json().catch(() => ({}));
-          throw new Error(errData.error?.message || errData.error || 'TTS failed');
-        }
-
-        const data = await response.json();
-        playFromUrl(data.audioUrl, messageId);
+        playFromUrl(audioUrl, messageId, controller.signal);
       };
 
       /**
@@ -311,22 +302,26 @@ export function useTTS(options: UseTTSOptions = {}) {
        * forever, and the person notices nothing.
        */
       if (audioUrl) {
-        playFromUrl(audioUrl, messageId, () => {
-          void synthesize().catch((e: unknown) => setError(errorMessage(e, 'Failed to read aloud')));
+        playFromUrl(audioUrl, messageId, controller.signal, () => {
+          void synthesize().catch((e: unknown) => {
+            if (!controller.signal.aborted) setError(errorMessage(e, 'Failed to read aloud'));
+          });
         });
         return;
       }
 
       await synthesize();
     } catch (e: unknown) {
+      if (controller.signal.aborted) return;
       console.error('[TTS] Error:', e);
       setError(errorMessage(e, 'Failed to read aloud'));
     }
-  }, [getToken, apiUrl, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
+  }, [getToken, apiUrl, voiceModel, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
 
   // Cleanup on unmount
   useEffect(() => {
     return () => {
+      requestRef.current?.abort();
       releasePlayer();
     };
   }, []);