@alia.onl/sdk 10.1.1 → 11.0.0

package/src/hooks/useAliaChat.ts

@@ -1,7 +1,7 @@
 import React, { useCallback, useEffect, useRef, useState } from 'react';
 import { useOxy } from '@oxy.so/services/ui/client';
 import type { ChatMessage, ToolInvocation } from '../types';
-import { getTextFromContent } from '../lib/content-utils';
+import { getTextFromContent, type MessageContent } from '../lib/content-utils';
 import { resolveModelId } from '../lib/catalogue';
 import { streamAliaChat } from '../lib/chat-transport';
 import type { AliaChatStreamEvent } from '../lib/chat-stream';
@@ -45,10 +45,15 @@ export interface UseAliaChatOptions {
   accessToken?: string;
 }
 
+export interface SendOptions {
+  images?: ReadonlyArray<string>;
+}
+
 export interface UseAliaChatReturn {
   messages: ChatMessage[];
   setMessages: React.Dispatch<React.SetStateAction<ChatMessage[]>>;
-  send: (text: string) => void;
+  /** `images` are `data:` or `https:` URLs sent with the turn as `image_url` parts. */
+  send: (text: string, options?: SendOptions) => void;
   isStreaming: boolean;
   stop: () => void;
   clear: () => void;
@@ -112,12 +117,7 @@ function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>
  * handler during its recorded compatibility window.
  */
 export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn {
-  const {
-    apiUrl = API_URL,
-    model,
-    clientContext,
-    accessToken: accessTokenProp,
-  } = options;
+  const { apiUrl = API_URL, model, clientContext, accessToken: accessTokenProp } = options;
   const requestUrl = '/v1/chat/completions';
 
   const { oxyServices } = useOxy();
@@ -231,9 +231,10 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
   }, [cancelPendingTimer]);
 
   const send = useCallback(
-    async (text: string) => {
+    async (text: string, options?: SendOptions) => {
       const trimmed = text.trim();
-      if (!trimmed) return;
+      const images = options?.images ?? [];
+      if (!trimmed && images.length === 0) return;
 
       const activeToken = oxyServices.session.accessToken;
       if (accessTokenProp !== undefined && accessTokenProp !== activeToken) {
@@ -253,10 +254,17 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
         flushPendingUpdates(previousPending);
       }
 
+      const userContent: MessageContent =
+        images.length > 0
+          ? [
+              ...(trimmed ? [{ type: 'text', text: trimmed }] : []),
+              ...images.map((url) => ({ type: 'image_url', image_url: { url } })),
+            ]
+          : trimmed;
       const userMessage: ChatMessage = {
         id: generateId(),
         role: 'user',
-        content: trimmed,
+        content: userContent,
         createdAt: Date.now(),
       };
       const assistantMessage: ChatMessage = {
@@ -281,7 +289,7 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
       setError(null);
       setIsStreaming(true);
 
-      const apiMessages: Array<{ role: string; content: string }> = [];
+      const apiMessages: Array<{ role: string; content: MessageContent }> = [];
       if (clientContext) apiMessages.push({ role: 'system', content: clientContext });
       for (const message of history) {
         if (message.role === 'system') continue;
@@ -295,7 +303,7 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
           }
         }
       }
-      apiMessages.push({ role: 'user', content: trimmed });
+      apiMessages.push({ role: 'user', content: userContent });
 
       const pending: PendingUpdates = {
         assistantId: assistantMessage.id,
@@ -351,8 +359,7 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
             updateAssistant(pending.assistantId, (last) => ({
               researchProgress: {
                 ...event.progress,
-                subQuestions:
-                  event.progress.subQuestions ?? last.researchProgress?.subQuestions,
+                subQuestions: event.progress.subQuestions ?? last.researchProgress?.subQuestions,
               },
             }));
             return;

package/src/hooks/useAmbientWave.ts

@@ -60,7 +60,12 @@ const ACTIVE_INTENSITY = 0.35;
  * reanimated-web; the combiner is a pure-read `useDerivedValue`. Effect-free — no
  * React `useEffect`.
  */
-export function useAmbientWave({ voice, isTTSPlaying, ttsWaveAmplitude, isGenerating }: UseAmbientWaveOptions): UseAmbientWaveResult {
+export function useAmbientWave({
+  voice,
+  isTTSPlaying,
+  ttsWaveAmplitude,
+  isGenerating,
+}: UseAmbientWaveOptions): UseAmbientWaveResult {
   // ── Idle breath: subtle 0.04↔0.09 swell (~7s), started once on mount ──
   const idleBreath = useSharedValue(0.04);
   useAnimatedReaction(
@@ -131,14 +136,26 @@ export function useAmbientWave({ voice, isTTSPlaying, ttsWaveAmplitude, isGenera
   // ── Combined amplitude — pure reads only (ticks on web) ──
   const voiceAmplitude = voice?.waveAmplitude;
   const waveAmplitude = useDerivedValue(
-    () => Math.max(idleBreath.value, voiceAmplitude ? voiceAmplitude.value : 0, ttsWaveAmplitude.value, stt.value, thinkingAmp.value),
+    () =>
+      Math.max(
+        idleBreath.value,
+        voiceAmplitude ? voiceAmplitude.value : 0,
+        ttsWaveAmplitude.value,
+        stt.value,
+        thinkingAmp.value,
+      ),
     [voiceAmplitude, ttsWaveAmplitude],
   );
 
   // ── Mode → palette state + overlay intensity (plain derived) ──
   return useMemo<UseAmbientWaveResult>(() => {
     if (voice?.isActive && voice.isConnected) {
-      return { waveAmplitude, agentState: voice.agentState, intensity: ACTIVE_INTENSITY, mode: 'voice' };
+      return {
+        waveAmplitude,
+        agentState: voice.agentState,
+        intensity: ACTIVE_INTENSITY,
+        mode: 'voice',
+      };
     }
     if (isTTSPlaying) {
       return { waveAmplitude, agentState: 'speaking', intensity: ACTIVE_INTENSITY, mode: 'tts' };
@@ -147,8 +164,21 @@ export function useAmbientWave({ voice, isTTSPlaying, ttsWaveAmplitude, isGenera
       return { waveAmplitude, agentState: 'listening', intensity: ACTIVE_INTENSITY, mode: 'stt' };
     }
     if (isGenerating) {
-      return { waveAmplitude, agentState: 'thinking', intensity: THINKING_INTENSITY, mode: 'thinking' };
+      return {
+        waveAmplitude,
+        agentState: 'thinking',
+        intensity: THINKING_INTENSITY,
+        mode: 'thinking',
+      };
     }
     return { waveAmplitude, agentState: 'idle', intensity: IDLE_INTENSITY, mode: 'idle' };
-  }, [waveAmplitude, voice?.isActive, voice?.isConnected, voice?.agentState, isTTSPlaying, sttIsRecording, isGenerating]);
+  }, [
+    waveAmplitude,
+    voice?.isActive,
+    voice?.isConnected,
+    voice?.agentState,
+    isTTSPlaying,
+    sttIsRecording,
+    isGenerating,
+  ]);
 }

package/src/hooks/useSoundEffects.ts

@@ -70,18 +70,21 @@ export function useSoundEffects(enabled: boolean = false, sounds: SoundSources =
     };
   }, [enabled, sounds]);
 
-  const play = useCallback((sound: SoundName) => {
-    if (!enabled) return;
-    const player = playersRef.current.get(sound);
-    if (player) {
-      try {
-        player.seekTo(0);
-        player.play();
-      } catch {
-        // Playback failed — ignore
+  const play = useCallback(
+    (sound: SoundName) => {
+      if (!enabled) return;
+      const player = playersRef.current.get(sound);
+      if (player) {
+        try {
+          player.seekTo(0);
+          player.play();
+        } catch {
+          // Playback failed — ignore
+        }
       }
-    }
-  }, [enabled]);
+    },
+    [enabled],
+  );
 
   return { play };
 }

package/src/hooks/useSpeechToText.ts

@@ -8,7 +8,11 @@ import {
 } from '../lib/speech-recognition';
 import type { SpeechRecognitionSession } from '../lib/speech-recognition-types';
 import type { SpeechRecognizerChoice } from '../lib/speech-recognition-types';
-import { speechFailureCode, VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
+import {
+  speechFailureCode,
+  VOICE_ERROR_MESSAGES,
+  type VoiceErrorCode,
+} from '../lib/speech-messages';
 import { defaultSpeechLanguage } from '../lib/speech-language';
 
 // ============== OPTIONS ==============
@@ -81,7 +85,10 @@ export function useSpeechToText(options: UseSTTOptions = {}) {
   const currentTextRef = useRef('');
   const restartsRef = useRef(0);
   /** The recognizer and tag this dictation uses, chosen once when it starts. */
-  const recognizerRef = useRef<{ lang: string; service?: string; silent: boolean }>({ lang, silent: false });
+  const recognizerRef = useRef<{ lang: string; service?: string; silent: boolean }>({
+    lang,
+    silent: false,
+  });
   const endWaitersRef = useRef<Array<() => void>>([]);
   const lastMeteringRef = useRef(0);
   const mountedRef = useRef(true);

package/src/hooks/useTTS.ts

@@ -146,177 +146,184 @@ export function useTTS(options: UseTTSOptions = {}) {
    * gets to rebuild rather than the person getting an error about a button they
    * pressed twice.
    */
-  const playFromUrl = useCallback((
-    audioUrl: string,
-    _messageId: string,
-    signal: AbortSignal,
-    onUnplayable?: () => void,
-  ) => {
-    releasePlayer();
+  const playFromUrl = useCallback(
+    (audioUrl: string, _messageId: string, signal: AbortSignal, onUnplayable?: () => void) => {
+      releasePlayer();
 
-    (async () => {
-      try {
-        const { createAudioPlayer } = await import('expo-audio');
-        if (signal.aborted) return;
-        /**
-         * `crossOrigin` is web-only and ignored elsewhere, and it is what makes
-         * the waveform readable there: a browser will not let an `AnalyserNode`
-         * see a cross-origin media element loaded without CORS, so without this
-         * `setAudioSamplingEnabled` below silently declines and the field never
-         * moves. Clips are served by `GET /media` on the API host, which is a
-         * different origin from the app on every deployment.
-         *
-         * It costs the clip its CORS headers if the app's origin is not on the
-         * API's allowlist — but such an origin cannot load a conversation to
-         * read aloud in the first place, since `/media` and `/conversations`
-         * share that allowlist. Measured against production 2026-08-26:
-         * `Origin: https://alia.onl` is answered
-         * `access-control-allow-origin: https://alia.onl`, with `Vary: Origin`.
-         */
-        const player = createAudioPlayer({ uri: audioUrl }, { crossOrigin: 'anonymous' });
-        playerRef.current = player;
+      (async () => {
+        try {
+          const { createAudioPlayer } = await import('expo-audio');
+          if (signal.aborted) return;
+          /**
+           * `crossOrigin` is web-only and ignored elsewhere, and it is what makes
+           * the waveform readable there: a browser will not let an `AnalyserNode`
+           * see a cross-origin media element loaded without CORS, so without this
+           * `setAudioSamplingEnabled` below silently declines and the field never
+           * moves. Clips are served by `GET /media` on the API host, which is a
+           * different origin from the app on every deployment.
+           *
+           * It costs the clip its CORS headers if the app's origin is not on the
+           * API's allowlist — but such an origin cannot load a conversation to
+           * read aloud in the first place, since `/media` and `/conversations`
+           * share that allowlist. Measured against production 2026-08-26:
+           * `Origin: https://alia.onl` is answered
+           * `access-control-allow-origin: https://alia.onl`, with `Vary: Origin`.
+           */
+          const player = createAudioPlayer({ uri: audioUrl }, { crossOrigin: 'anonymous' });
+          playerRef.current = player;
+
+          /**
+           * The real waveform, which is what the ambient field behind the
+           * conversation moves to. Enabled BEFORE `play()`: on web this is what
+           * builds the analyser, and the sampling loop only starts if the player
+           * is already playing or starts afterwards.
+           *
+           * It can decline, and declining is not an error. Android routes
+           * playback sampling through `android.media.audiofx.Visualizer`, which
+           * needs `RECORD_AUDIO` already granted — and asking a person for the
+           * microphone so that a background can move would be a worse trade than
+           * the background not moving. Where it declines no buffer ever arrives,
+           * the level stays at zero and the field simply rests. That is the
+           * honest reading of "we cannot hear this", and it is deliberately not
+           * papered over with motion that does not come from the audio.
+           */
+          player.setAudioSamplingEnabled(true);
+          const meter = createAudioLevelMeter();
+          player.addListener('audioSampleUpdate', (sample: AudioSample) => {
+            const frames = sample.channels[0]?.frames;
+            if (frames === undefined) return;
+            ttsWaveAmplitude.value = meter.push(frames, Date.now());
+          });
 
-        /**
-         * The real waveform, which is what the ambient field behind the
-         * conversation moves to. Enabled BEFORE `play()`: on web this is what
-         * builds the analyser, and the sampling loop only starts if the player
-         * is already playing or starts afterwards.
-         *
-         * It can decline, and declining is not an error. Android routes
-         * playback sampling through `android.media.audiofx.Visualizer`, which
-         * needs `RECORD_AUDIO` already granted — and asking a person for the
-         * microphone so that a background can move would be a worse trade than
-         * the background not moving. Where it declines no buffer ever arrives,
-         * the level stays at zero and the field simply rests. That is the
-         * honest reading of "we cannot hear this", and it is deliberately not
-         * papered over with motion that does not come from the audio.
-         */
-        player.setAudioSamplingEnabled(true);
-        const meter = createAudioLevelMeter();
-        player.addListener('audioSampleUpdate', (sample: AudioSample) => {
-          const frames = sample.channels[0]?.frames;
-          if (frames === undefined) return;
-          ttsWaveAmplitude.value = meter.push(frames, Date.now());
-        });
-
-        player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
-          if (signal.aborted || playerRef.current !== player) return;
-          if (status.error) {
-            releasePlayer();
-            if (onUnplayable) {
-              onUnplayable();
+          player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
+            if (signal.aborted || playerRef.current !== player) return;
+            if (status.error) {
+              releasePlayer();
+              if (onUnplayable) {
+                onUnplayable();
+                return;
+              }
+              setError(status.error);
               return;
             }
-            setError(status.error);
+            if (status.didJustFinish) {
+              releasePlayer();
+              reset();
+            }
+          });
+
+          player.play();
+          setPlaybackState('playing');
+        } catch {
+          if (signal.aborted) return;
+          if (onUnplayable) {
+            onUnplayable();
             return;
           }
-          if (status.didJustFinish) {
-            releasePlayer();
-            reset();
-          }
-        });
-
-        player.play();
-        setPlaybackState('playing');
-      } catch {
-        if (signal.aborted) return;
-        if (onUnplayable) {
-          onUnplayable();
+          setError('Audio playback not available');
+        }
+      })();
+    },
+    [releasePlayer, reset, setPlaybackState, setError],
+  );
+
+  const readAloud = useCallback(
+    async (messageId: string, text: string, conversationId?: string, audioUrl?: string) => {
+      // Read volatile playback state at call time (not via closure) so this
+      // callback's identity survives every playback transition. It is passed to
+      // memoized message rows — an unstable identity re-renders all of them.
+      const { activeMessageId, playbackState } = useTTSStore.getState();
+
+      // If same message is playing, toggle pause/play
+      if (activeMessageId === messageId) {
+        if (playbackState === 'playing') {
+          playerRef.current?.pause();
+          setPlaybackState('paused');
+          return;
+        }
+        if (playbackState === 'paused') {
+          playerRef.current?.play();
+          setPlaybackState('playing');
           return;
         }
-        setError('Audio playback not available');
-      }
-    })();
-  }, [releasePlayer, reset, setPlaybackState, setError]);
-
-  const readAloud = useCallback(async (
-    messageId: string,
-    text: string,
-    conversationId?: string,
-    audioUrl?: string,
-  ) => {
-    // Read volatile playback state at call time (not via closure) so this
-    // callback's identity survives every playback transition. It is passed to
-    // memoized message rows — an unstable identity re-renders all of them.
-    const { activeMessageId, playbackState } = useTTSStore.getState();
-
-    // If same message is playing, toggle pause/play
-    if (activeMessageId === messageId) {
-      if (playbackState === 'playing') {
-        playerRef.current?.pause();
-        setPlaybackState('paused');
-        return;
       }
-      if (playbackState === 'paused') {
-        playerRef.current?.play();
-        setPlaybackState('playing');
-        return;
+
+      // Stop any current playback
+      if (activeMessageId) {
+        stop();
       }
-    }
 
-    // Stop any current playback
-    if (activeMessageId) {
-      stop();
-    }
+      const controller = new AbortController();
+      requestRef.current = controller;
+      try {
+        setActiveMessage(messageId);
+        setPlaybackState('loading');
 
-    const controller = new AbortController();
-    requestRef.current = controller;
-    try {
-      setActiveMessage(messageId);
-      setPlaybackState('loading');
-
-      /**
-       * Ask the server to make it, and play what comes back.
-       *
-       * Hoisted out of the cached branch below so a clip that has aged out of
-       * storage can fall back to it. `playFromUrl` is called WITHOUT a fallback
-       * here: a freshly made clip that will not play is a real failure, and
-       * retrying it would be a loop.
-       */
-      const synthesize = async () => {
-        const token = getToken();
-        if (!token) {
-          throw new Error('Not authenticated');
-        }
+        /**
+         * Ask the server to make it, and play what comes back.
+         *
+         * Hoisted out of the cached branch below so a clip that has aged out of
+         * storage can fall back to it. `playFromUrl` is called WITHOUT a fallback
+         * here: a freshly made clip that will not play is a real failure, and
+         * retrying it would be a loop.
+         */
+        const synthesize = async () => {
+          const token = getToken();
+          if (!token) {
+            throw new Error('Not authenticated');
+          }
 
-        const audioUrl = await requestSpeechClip({
-          apiUrl,
-          token,
-          model: voiceModel,
-          input: text,
-          voice: getTTSVoice(),
-          speed: getTTSSpeed(),
-          conversationId,
-          messageId,
-          signal: controller.signal,
-        });
-        playFromUrl(audioUrl, messageId, controller.signal);
-      };
-
-      /**
-       * A stored clip plays straight away, and is remade if it has gone.
-       *
-       * The row keeps its key long after a `tts/` object ages out of storage,
-       * so the link is signed, valid, and points at nothing — a failure only
-       * the player ever sees. Remaking it is cheaper than keeping every clip
-       * forever, and the person notices nothing.
-       */
-      if (audioUrl) {
-        playFromUrl(audioUrl, messageId, controller.signal, () => {
-          void synthesize().catch((e: unknown) => {
-            if (!controller.signal.aborted) setError(errorMessage(e, 'Failed to read aloud'));
+          const audioUrl = await requestSpeechClip({
+            apiUrl,
+            token,
+            model: voiceModel,
+            input: text,
+            voice: getTTSVoice(),
+            speed: getTTSSpeed(),
+            conversationId,
+            messageId,
+            signal: controller.signal,
           });
-        });
-        return;
-      }
+          playFromUrl(audioUrl, messageId, controller.signal);
+        };
 
-      await synthesize();
-    } catch (e: unknown) {
-      if (controller.signal.aborted) return;
-      console.error('[TTS] Error:', e);
-      setError(errorMessage(e, 'Failed to read aloud'));
-    }
-  }, [getToken, apiUrl, voiceModel, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
+        /**
+         * A stored clip plays straight away, and is remade if it has gone.
+         *
+         * The row keeps its key long after a `tts/` object ages out of storage,
+         * so the link is signed, valid, and points at nothing — a failure only
+         * the player ever sees. Remaking it is cheaper than keeping every clip
+         * forever, and the person notices nothing.
+         */
+        if (audioUrl) {
+          playFromUrl(audioUrl, messageId, controller.signal, () => {
+            void synthesize().catch((e: unknown) => {
+              if (!controller.signal.aborted) setError(errorMessage(e, 'Failed to read aloud'));
+            });
+          });
+          return;
+        }
+
+        await synthesize();
+      } catch (e: unknown) {
+        if (controller.signal.aborted) return;
+        console.error('[TTS] Error:', e);
+        setError(errorMessage(e, 'Failed to read aloud'));
+      }
+    },
+    [
+      getToken,
+      apiUrl,
+      voiceModel,
+      getTTSVoice,
+      getTTSSpeed,
+      stop,
+      playFromUrl,
+      setActiveMessage,
+      setPlaybackState,
+      setError,
+    ],
+  );
 
   // Cleanup on unmount
   useEffect(() => {