@alia.onl/sdk 13.2.0 → 15.3.0

package/src/components/voice/VoiceControls.tsx

@@ -8,15 +8,17 @@ import { Loading } from '@oxy.so/bloom/loading';
 import { Text } from '@oxy.so/bloom/typography';
 import { VoiceControls as BloomVoiceControls, type VoiceState } from '@oxy.so/bloom/voice';
 import { View } from 'react-native';
-import type { RoomState, AgentState } from '../../types';
+import type { RoomState, AgentState, VoiceCallState } from '../../types';
 
 /** Every word the bar shows or announces. An app in another language passes its own. */
 export interface VoiceControlsLabels {
   connecting: string;
+  reconnecting: string;
   connected: string;
   listening: string;
   muted: string;
   thinking: string;
+  usingTool: string;
   speaking: string;
   mute: string;
   unmute: string;
@@ -25,10 +27,12 @@ export interface VoiceControlsLabels {
 
 export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
   connecting: 'Connecting...',
+  reconnecting: 'Reconnecting...',
   connected: 'Connected',
   listening: 'Listening...',
   muted: 'Muted',
   thinking: 'Thinking...',
+  usingTool: 'Working on it...',
   speaking: 'Speaking...',
   mute: 'Mute',
   unmute: 'Unmute',
@@ -41,6 +45,16 @@ export interface VoiceControlsProps {
   isMuted: boolean;
   onToggleMute: () => void;
   onEnd: () => void;
+  /**
+   * Something the person should know while the call listens — e.g. that the
+   * call is about to reach its time limit. Shown in place of "Listening…".
+   */
+  notice?: string | null;
+  /**
+   * The call's full state, when the host has it: a tool running reads as such
+   * rather than as "Thinking…".
+   */
+  callState?: VoiceCallState;
   /** Overrides for any of {@link VOICE_CONTROLS_LABELS}. */
   labels?: Partial<VoiceControlsLabels>;
 }
@@ -49,11 +63,15 @@ function getStatusText(
   roomState: RoomState,
   agentState: AgentState,
   isMuted: boolean,
+  notice: string | null,
   labels: VoiceControlsLabels,
+  callState: VoiceCallState | undefined,
 ): string {
   if (roomState === 'connecting') return labels.connecting;
+  if (roomState === 'reconnecting') return labels.reconnecting;
   if (roomState === 'connected') {
-    if (agentState === 'listening') return isMuted ? labels.muted : labels.listening;
+    if (agentState === 'listening') return isMuted ? labels.muted : (notice ?? labels.listening);
+    if (callState?.kind === 'using-tool') return labels.usingTool;
     if (agentState === 'thinking') return labels.thinking;
     if (agentState === 'speaking') return labels.speaking;
     return labels.connected;
@@ -74,10 +92,12 @@ export function VoiceControls({
   isMuted,
   onToggleMute,
   onEnd,
+  notice = null,
+  callState,
   labels: labelOverrides,
 }: VoiceControlsProps) {
   const labels = { ...VOICE_CONTROLS_LABELS, ...labelOverrides };
-  const statusText = getStatusText(roomState, agentState, isMuted, labels);
+  const statusText = getStatusText(roomState, agentState, isMuted, notice, labels, callState);
 
   return (
     <View className="items-center gap-4 pb-6 pt-4">
@@ -102,7 +122,11 @@ export function VoiceControls({
         />
       ) : null}
 
-      {roomState === 'connecting' ? <Loading accessibilityLabel={labels.connecting} /> : null}
+      {roomState === 'connecting' || roomState === 'reconnecting' ? (
+        <Loading
+          accessibilityLabel={roomState === 'connecting' ? labels.connecting : labels.reconnecting}
+        />
+      ) : null}
     </View>
   );
 }

package/src/components/voice/VoiceSession.tsx

@@ -2,7 +2,7 @@
  * A live voice call, mounted only while voice mode is on.
  *
  * The chat shell receives it as the `voiceSession` prop rather than importing
- * it, so a text-only consumer never compiles the voice loop, speech playback or
+ * it, so a text-only consumer never compiles the call, its audio engine or
  * the call UI — which is the reason the `@alia.onl/sdk/voice` entry exists.
  *
  * Mounting starts the call; unmounting ends it (`useVoiceRoom` stops listening
@@ -17,9 +17,18 @@ import { useAudioLevels } from '../../hooks/useAudioLevels';
 import { VoiceControls } from './VoiceControls';
 import type { VoiceSessionProps } from '../../types';
 
-export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps) {
+export function VoiceSession({
+  apiUrl,
+  conversationId,
+  agentId,
+  locale,
+  voice,
+  onStateChange,
+  onEnd,
+}: VoiceSessionProps) {
   const {
     room,
+    callState,
     roomState,
     agentState,
     isMuted,
@@ -28,7 +37,13 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
     isConnected,
     connect,
     toggleMute,
-  } = useVoiceRoom({ apiUrl });
+  } = useVoiceRoom({
+    apiUrl,
+    ...(conversationId === undefined ? {} : { conversationId }),
+    ...(agentId === undefined ? {} : { agentId }),
+    ...(locale === undefined ? {} : { lang: locale }),
+    ...(voice === undefined ? {} : { voicePreference: voice }),
+  });
 
   const { captureLevel, playbackLevel } = useAudioLevelMonitor(room, isConnected);
   const { waveAmplitude } = useAudioLevels({
@@ -44,21 +59,21 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
   useEffect(() => {
     if (dialedRef.current) return;
     dialedRef.current = true;
-    connect();
+    void connect();
   }, [connect]);
 
   useEffect(() => {
     onStateChange({ isConnected, agentState, waveAmplitude, messages });
   }, [isConnected, agentState, waveAmplitude, messages, onStateChange]);
 
-  // End the call on failure, or once a room we actually reached goes away. The
+  // End the call on failure, or once a call we actually reached ends. The
   // "reached" flag is what keeps the pre-connect `disconnected` state — which
   // every session starts in — from ending the call before it begins.
   const reachedRoomRef = useRef(false);
   useEffect(() => {
-    if (roomState === 'connected') reachedRoomRef.current = true;
+    if (roomState !== 'disconnected') reachedRoomRef.current = true;
     if (error || roomState === 'error') {
-      onEnd(error ?? VOICE_ERROR_MESSAGES['turn-failed']);
+      onEnd(error ?? VOICE_ERROR_MESSAGES['realtime-failed']);
       return;
     }
     if (roomState === 'disconnected' && reachedRoomRef.current) onEnd();
@@ -71,6 +86,7 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
       isMuted={isMuted}
       onToggleMute={toggleMute}
       onEnd={onEnd}
+      callState={callState}
     />
   );
 }

package/src/hooks/useAliaActivity.ts

@@ -0,0 +1,31 @@
+import { useEffect } from 'react';
+import { useOxy } from '@oxy.so/services/ui/client';
+import type { AliaAvatarState } from '../components/AliaAvatar';
+import { selectAliaActivity, useAliaActivityStore } from '../lib/activity-store';
+import { acquireActivitySocket } from '../lib/activity-socket';
+import { ALIA_API_URL } from '../lib/api-url';
+
+export interface UseAliaActivityOptions {
+  /** The API the activity socket connects to. Defaults like `useAliaChat`'s `apiUrl`. */
+  apiUrl?: string;
+}
+
+/**
+ * What Alia is doing right now, for any surface that wants to show it — an "Ask Alia" button's
+ * avatar, a status dot. It is the busiest of:
+ *
+ * - a turn running in this app (`useAliaChat`, so the sheet drives every button), and
+ * - her runs for the signed-in person elsewhere, reported by the server over one shared socket
+ *   (background work, another device), connected only while someone is signed in AND using this.
+ */
+export function useAliaActivity(options: UseAliaActivityOptions = {}): AliaAvatarState {
+  const { apiUrl = ALIA_API_URL } = options;
+  const { oxyServices, isAuthenticated } = useOxy();
+
+  useEffect(() => {
+    if (!isAuthenticated) return;
+    return acquireActivitySocket(apiUrl, () => oxyServices.session.accessToken);
+  }, [apiUrl, isAuthenticated, oxyServices]);
+
+  return useAliaActivityStore(selectAliaActivity);
+}

package/src/hooks/useAliaChat.ts

@@ -1,4 +1,4 @@
-import React, { useCallback, useEffect, useRef, useState } from 'react';
+import React, { useCallback, useEffect, useId, useRef, useState } from 'react';
 import { useOxy } from '@oxy.so/services/ui/client';
 import type { ChatMessage, ToolInvocation } from '../types';
 import { getTextFromContent, type MessageContent } from '../lib/content-utils';
@@ -7,7 +7,9 @@ import { agentMessageId, delegatedAgent } from '../lib/delegation';
 import { streamAliaChat } from '../lib/chat-transport';
 import type { AliaChatStreamEvent } from '../lib/chat-stream';
 
-const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+import { ALIA_API_URL } from '../lib/api-url';
+import { useAliaActivityStore } from '../lib/activity-store';
+import { turnActivityState } from '../lib/turn-state';
 
 export interface UseAliaChatOptions {
   /**
@@ -129,7 +131,7 @@ function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>
  * handler during its recorded compatibility window.
  */
 export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn {
-  const { apiUrl = API_URL, model, clientContext, accessToken: accessTokenProp } = options;
+  const { apiUrl = ALIA_API_URL, model, clientContext, accessToken: accessTokenProp } = options;
   const directoryRef = useRef(new Map<string, { id: string; character?: unknown }>());
   directoryRef.current = new Map((options.agents ?? []).map((agent) => [agent.id, agent]));
   const requestUrl = '/v1/chat/completions';
@@ -147,6 +149,15 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
     messagesRef.current = messages;
   }, [messages]);
 
+  // The turn's lifecycle, published for every "Ask Alia" button (`useAliaActivity`).
+  const activitySource = useId();
+  const activity = turnActivityState(messages, isStreaming);
+  useEffect(() => {
+    const { setLocal } = useAliaActivityStore.getState();
+    setLocal(activitySource, activity);
+    return () => setLocal(activitySource, 'idle');
+  }, [activitySource, activity]);
+
   const updateAssistant = useCallback(
     (assistantId: string, updater: (message: ChatMessage) => Partial<ChatMessage>) => {
       if (!mountedRef.current) return;

package/src/hooks/useAmbientWave.ts

@@ -10,7 +10,7 @@ import {
   Easing,
   type SharedValue,
 } from 'react-native-reanimated';
-import { useSTTStore } from './useSpeechToText';
+import { useDictationStore } from './useDictation';
 import { LEVEL_ATTACK_MS, LEVEL_RELEASE_MS } from '../lib/audio-level';
 import type { AgentState } from '../types';
 
@@ -95,8 +95,8 @@ export function useAmbientWave({
   // travel takes about three times as long. Tuning the constant moves both, by
   // different amounts. They are shared because one number should tune one
   // feel, not because the two are interchangeable.
-  const sttIsRecording = useSTTStore((s) => s.isRecording);
-  const sttMetering = useSTTStore((s) => s.metering);
+  const sttIsRecording = useDictationStore((s) => s.isRecording);
+  const sttMetering = useDictationStore((s) => s.metering);
   const stt = useSharedValue(0);
   useAnimatedReaction(
     () => (sttIsRecording ? Math.max(0.08, sttMetering) : -1),

package/src/hooks/useAudioLevelMonitor.ts

@@ -1,8 +1,8 @@
 /**
  * A voice call's live levels, as React state at ~20fps.
  *
- * `useVoiceRoom` writes the microphone level (from the recognizer) and the
- * playback level (from the clip being spoken) into a `VoiceLevelSource`; this
+ * `useVoiceRoom` writes the microphone level (from the captured buffers) and the
+ * playback level (from the answer being played) into a `VoiceLevelSource`; this
  * reads them on a clock so a wave can follow them without rendering once per
  * audio buffer. It used to put Web Audio analysers on a LiveKit room's tracks;
  * the signature kept its shape — `room` is now the level source `useVoiceRoom`

package/src/hooks/useDictation.ts

@@ -0,0 +1,295 @@
+/**
+ * The composer's microphone: record a clip, then transcribe it with a Kaana
+ * transcription model through Alia (`POST /v1/audio/transcriptions`, ADR 0017
+ * §7). Free on every plan, five minutes per clip at most.
+ *
+ *   idle → requesting-permission → recording → transcribing → idle (text)
+ *                                                           ↘ error { code }
+ *
+ * - `recording` is entered only once the recorder reports it started; the
+ *   level meter is the real recorder's.
+ * - The text arrives after the person stops — `onText` — and never partially:
+ *   a failed transcription inserts nothing and keeps the clip for ONE `retry()`.
+ * - At five minutes the recording stops by itself and is transcribed; the
+ *   remaining time is exposed for the last 30 seconds.
+ *
+ * Nothing is recognized on the device.
+ */
+
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { create } from 'zustand';
+import { useOxy } from '@oxy.so/services/ui/client';
+import {
+  dictationFailureCode,
+  MAX_DICTATION_MS,
+  requestTranscription,
+  type DictationClip,
+} from '../lib/dictation';
+import { DictationRecorderError, useDictationRecorder } from '../lib/dictation-recorder';
+import { defaultSpeechLanguage } from '../lib/speech-language';
+import { VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
+
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
+
+/** The remaining time is shown from here on. */
+const COUNTDOWN_FROM_MS = 30_000;
+const CLOCK_MS = 250;
+
+export type DictationState =
+  | 'idle'
+  | 'requesting-permission'
+  | 'recording'
+  | 'transcribing'
+  | 'error';
+
+export interface UseDictationOptions {
+  apiUrl?: string;
+  /** Bearer; defaults to the surrounding Oxy session's. */
+  accessToken?: string;
+  /** Language spoken (BCP-47); defaults to the platform's. The Alia app passes its own. */
+  lang?: string;
+  /** The transcribed text, once — after a stop, the five-minute stop, or a retry. */
+  onText?: (text: string) => void;
+}
+
+interface DictationStoreState {
+  isRecording: boolean;
+  metering: number;
+  setRecording: (isRecording: boolean) => void;
+  setMetering: (metering: number) => void;
+}
+
+/**
+ * Whether the composer is recording, and its level: what the ambient field and
+ * the app's turn actions read without holding the hook.
+ */
+export const useDictationStore = create<DictationStoreState>((set) => ({
+  isRecording: false,
+  metering: 0,
+  setRecording: (isRecording) => set({ isRecording }),
+  setMetering: (metering) => set({ metering }),
+}));
+
+export function useDictation(options: UseDictationOptions = {}) {
+  const { oxyServices } = useOxy();
+  const recorder = useDictationRecorder();
+  const [state, setStateValue] = useState<DictationState>('idle');
+  const [errorCode, setErrorCode] = useState<VoiceErrorCode | null>(null);
+  const [elapsedMs, setElapsedMs] = useState(0);
+  const [canRetry, setCanRetry] = useState(false);
+
+  const stateRef = useRef<DictationState>('idle');
+  const mountedRef = useRef(true);
+  const startedAtRef = useRef(0);
+  const clockRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const capRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const failedClipRef = useRef<DictationClip | null>(null);
+  const abortRef = useRef<AbortController | null>(null);
+  const lastMeteringRef = useRef(0);
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+
+  const setState = useCallback((next: DictationState) => {
+    stateRef.current = next;
+    if (mountedRef.current) setStateValue(next);
+  }, []);
+
+  const report = useCallback(
+    (code: VoiceErrorCode | null) => {
+      if (mountedRef.current) setErrorCode(code);
+      if (code !== null) setState('error');
+    },
+    [setState],
+  );
+
+  useEffect(() => {
+    const store = useDictationStore.getState();
+    store.setRecording(state === 'recording');
+    if (state !== 'recording') {
+      lastMeteringRef.current = 0;
+      store.setMetering(0);
+    }
+  }, [state]);
+
+  const clearClock = useCallback((): void => {
+    if (clockRef.current !== null) clearInterval(clockRef.current);
+    if (capRef.current !== null) clearTimeout(capRef.current);
+    clockRef.current = null;
+    capRef.current = null;
+  }, []);
+
+  const transcribe = useCallback(
+    async (clip: DictationClip): Promise<string | null> => {
+      const token = optionsRef.current.accessToken || oxyServices.session.accessToken || '';
+      if (token === '') {
+        failedClipRef.current = null;
+        if (mountedRef.current) setCanRetry(false);
+        report('not-authenticated');
+        return null;
+      }
+      setState('transcribing');
+      const controller = new AbortController();
+      abortRef.current = controller;
+      try {
+        const result = await requestTranscription({
+          apiUrl: optionsRef.current.apiUrl || API_URL,
+          token,
+          clip,
+          language: optionsRef.current.lang ?? defaultSpeechLanguage(),
+          signal: controller.signal,
+        });
+        if (controller.signal.aborted) return null;
+        failedClipRef.current = null;
+        if (mountedRef.current) setCanRetry(false);
+        setState('idle');
+        const text = result.text.trim();
+        if (text === '') return null;
+        optionsRef.current.onText?.(text);
+        return text;
+      } catch (caught: unknown) {
+        if (controller.signal.aborted) return null;
+        const code = dictationFailureCode(caught);
+        // Kept for one retry, unless retrying cannot change the answer.
+        const keep = code !== 'too-long' && code !== 'not-authenticated';
+        failedClipRef.current = keep ? clip : null;
+        if (mountedRef.current) setCanRetry(keep);
+        report(code);
+        return null;
+      } finally {
+        if (abortRef.current === controller) abortRef.current = null;
+      }
+    },
+    [oxyServices, report, setState],
+  );
+
+  const stop = useCallback(async (): Promise<string | null> => {
+    if (stateRef.current !== 'recording') return null;
+    clearClock();
+    setState('transcribing');
+    let clip: DictationClip | null;
+    try {
+      clip = await recorder.stop();
+    } catch {
+      report('mic-unavailable');
+      return null;
+    }
+    if (clip === null) {
+      setState('idle');
+      return null;
+    }
+    if (clip.durationMs > MAX_DICTATION_MS + 1_000) {
+      report('too-long');
+      return null;
+    }
+    return transcribe(clip);
+  }, [clearClock, recorder, report, setState, transcribe]);
+
+  const stopRef = useRef(stop);
+  stopRef.current = stop;
+
+  const start = useCallback(async (): Promise<void> => {
+    if (stateRef.current !== 'idle' && stateRef.current !== 'error') return;
+    if (mountedRef.current) {
+      setErrorCode(null);
+      setCanRetry(false);
+      setElapsedMs(0);
+    }
+    failedClipRef.current = null;
+    setState('requesting-permission');
+    try {
+      await recorder.start((level) => {
+        if (stateRef.current !== 'recording') return;
+        // Quiet should not thrash the store's subscribers.
+        if (Math.abs(level - lastMeteringRef.current) < 0.02) return;
+        lastMeteringRef.current = level;
+        useDictationStore.getState().setMetering(level);
+      });
+    } catch (caught: unknown) {
+      if ((stateRef.current as DictationState) !== 'requesting-permission') return;
+      report(caught instanceof DictationRecorderError ? caught.code : 'mic-unavailable');
+      return;
+    }
+    if ((stateRef.current as DictationState) !== 'requesting-permission') {
+      // Cancelled while the permission was asked.
+      recorder.cancel();
+      return;
+    }
+    startedAtRef.current = Date.now();
+    setState('recording');
+    clockRef.current = setInterval(() => {
+      if (mountedRef.current) setElapsedMs(Date.now() - startedAtRef.current);
+    }, CLOCK_MS);
+    capRef.current = setTimeout(() => {
+      void stopRef.current();
+    }, MAX_DICTATION_MS);
+  }, [recorder, report, setState]);
+
+  const retry = useCallback(async (): Promise<string | null> => {
+    const clip = failedClipRef.current;
+    if (clip === null || stateRef.current === 'recording' || stateRef.current === 'transcribing')
+      return null;
+    // One retry: whatever it answers, the clip is not kept for another.
+    failedClipRef.current = null;
+    if (mountedRef.current) {
+      setCanRetry(false);
+      setErrorCode(null);
+    }
+    const text = await transcribe(clip);
+    failedClipRef.current = null;
+    if (mountedRef.current) setCanRetry(false);
+    return text;
+  }, [transcribe]);
+
+  const cancel = useCallback(() => {
+    clearClock();
+    abortRef.current?.abort();
+    abortRef.current = null;
+    if (stateRef.current === 'recording' || stateRef.current === 'requesting-permission')
+      recorder.cancel();
+    failedClipRef.current = null;
+    if (mountedRef.current) {
+      setErrorCode(null);
+      setCanRetry(false);
+    }
+    setState('idle');
+  }, [clearClock, recorder, setState]);
+
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      clearClock();
+      abortRef.current?.abort();
+      const wasRecording =
+        stateRef.current === 'recording' || stateRef.current === 'requesting-permission';
+      stateRef.current = 'idle';
+      if (wasRecording) recorder.cancel();
+      // The store outlives this hook: a composer unmounted mid-dictation (the
+      // call bar replacing it) must not leave the field reading "recording".
+      useDictationStore.getState().setRecording(false);
+      useDictationStore.getState().setMetering(0);
+    };
+  }, [clearClock, recorder]);
+
+  const remaining = MAX_DICTATION_MS - elapsedMs;
+  return {
+    state,
+    /** What went wrong, in English. */
+    error: errorCode === null ? null : VOICE_ERROR_MESSAGES[errorCode],
+    /** The same, as a code an app can translate. */
+    errorCode,
+    /** Since the recording started. */
+    elapsedMs,
+    /** Left before the five-minute stop, in the last 30 seconds; else `null`. */
+    remainingMs:
+      state === 'recording' && remaining <= COUNTDOWN_FROM_MS ? Math.max(0, remaining) : null,
+    /** A failed transcription kept its clip for one more try. */
+    canRetry,
+    start,
+    stop,
+    retry,
+    cancel,
+    isRecording: state === 'recording',
+    isTranscribing: state === 'transcribing',
+  };
+}