@alia.onl/sdk 1.0.0 → 3.0.0

package/src/hooks/useAliaChat.ts

@@ -1,11 +1,11 @@
-import { useState, useRef, useCallback } from 'react';
+import React, { useState, useRef, useCallback, useEffect } from 'react';
 import { useOxy } from '@oxyhq/services';
 import type { ChatMessage, ToolInvocation } from '../types';
 
-const API_URL = process.env.EXPO_PUBLIC_API_URL;
+const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
 
 export interface UseAliaChatOptions {
-  /** Alia API base URL (default: EXPO_PUBLIC_API_URL) */
+  /** Alia API base URL (default: EXPO_PUBLIC_ALIA_API_URL or https://api.alia.onl) */
   apiUrl?: string;
   /** Alia model to use (default: 'alia-v1') */
   model?: string;
@@ -17,22 +17,27 @@ export interface UseAliaChatOptions {
 
 export interface UseAliaChatReturn {
   messages: ChatMessage[];
+  setMessages: React.Dispatch<React.SetStateAction<ChatMessage[]>>;
   send: (text: string) => void;
   isStreaming: boolean;
+  stop: () => void;
   clear: () => void;
   error: string | null;
 }
 
-let nextId = 0;
 function generateId(): string {
-  return `msg-${Date.now()}-${nextId++}`;
+  if (typeof crypto !== 'undefined' && crypto.randomUUID) {
+    return `msg-${crypto.randomUUID()}`;
+  }
+  return `msg-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
 }
 
 /**
  * SSE streaming chat hook for Alia.
  *
  * Sends messages to Alia's /v1/chat/completions endpoint and streams
- * responses back, including tool invocations.
+ * responses back, including tool invocations, reasoning, research progress,
+ * and plan previews.
  */
 export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn {
   const {
@@ -48,11 +53,98 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
   const [error, setError] = useState<string | null>(null);
   const abortRef = useRef<AbortController | null>(null);
 
+  // Use a ref to read current messages without putting it in send's dep array
+  const messagesRef = useRef<ChatMessage[]>([]);
+  useEffect(() => { messagesRef.current = messages; }, [messages]);
+
+  // Batching: accumulate streaming content and flush at ~20fps
+  const pendingContentRef = useRef('');
+  const pendingReasoningRef = useRef('');
+  const toolInvocationsRef = useRef<ToolInvocation[]>([]);
+  const flushTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  const flushPendingUpdates = useCallback(() => {
+    const content = pendingContentRef.current;
+    const reasoning = pendingReasoningRef.current;
+    const tools = toolInvocationsRef.current;
+    if (!content && !reasoning && tools.length === 0) return;
+
+    pendingContentRef.current = '';
+    pendingReasoningRef.current = '';
+
+    setMessages((prev) => {
+      const updated = [...prev];
+      const last = updated[updated.length - 1];
+      if (last?.role === 'assistant') {
+        const changes: Partial<ChatMessage> = {};
+        if (content) changes.content = last.content + content;
+        if (reasoning) changes.thinking = (last.thinking || '') + reasoning;
+        if (tools.length > 0) changes.toolInvocations = [...tools];
+        updated[updated.length - 1] = { ...last, ...changes };
+      }
+      return updated;
+    });
+  }, []);
+
+  const scheduleFlush = useCallback(() => {
+    if (flushTimerRef.current) return;
+    flushTimerRef.current = setTimeout(() => {
+      flushTimerRef.current = null;
+      flushPendingUpdates();
+    }, 50);
+  }, [flushPendingUpdates]);
+
+  // Cleanup flush timer on unmount
+  useEffect(() => {
+    return () => {
+      flushPendingUpdates();
+      if (flushTimerRef.current) {
+        clearTimeout(flushTimerRef.current);
+        flushTimerRef.current = null;
+      }
+    };
+  }, [flushPendingUpdates]);
+
   const getToken = useCallback((): string | null => {
     if (accessTokenProp) return accessTokenProp;
     return oxyServices.httpService.getAccessToken();
   }, [accessTokenProp, oxyServices]);
 
+  /** Update the last assistant message with partial changes (callback receives current message) */
+  const updateAssistant = useCallback(
+    (updater: (msg: ChatMessage) => Partial<ChatMessage>) => {
+      setMessages((prev) => {
+        const updated = [...prev];
+        const last = updated[updated.length - 1];
+        if (last?.role === 'assistant') {
+          updated[updated.length - 1] = { ...last, ...updater(last) };
+        }
+        return updated;
+      });
+    },
+    [],
+  );
+
+  /** Apply a tool result to the last assistant message and sync the ref */
+  const applyToolResult = useCallback(
+    (toolCallId: string, name: string | undefined, output: any) => {
+      updateAssistant((last) => {
+        const invocations = [...(last.toolInvocations || [])];
+        const idx = invocations.findIndex((t) => t.toolCallId === toolCallId);
+        if (idx >= 0) {
+          invocations[idx] = { ...invocations[idx], state: 'result', result: output };
+        } else {
+          invocations.push({ toolCallId, toolName: name || 'unknown', state: 'result', result: output });
+        }
+        return { toolInvocations: invocations };
+      });
+      toolInvocationsRef.current = toolInvocationsRef.current.map((t) =>
+        t.toolCallId === toolCallId ? { ...t, state: 'result' as const, result: output } : t
+      );
+    },
+    [updateAssistant],
+  );
+
   const send = useCallback(
     async (text: string) => {
       const trimmed = text.trim();
@@ -86,26 +178,40 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
       setMessages((prev) => [...prev, userMsg, assistantMsg]);
       setIsStreaming(true);
 
-      // Build messages payload for the API
+      // Build messages payload for the API (read from ref, not closure)
       const apiMessages: Array<{ role: string; content: string }> = [];
 
-      // System context (tells Alia which app the user is in)
       if (clientContext) {
         apiMessages.push({ role: 'system', content: clientContext });
       }
 
-      // Previous conversation history
-      for (const msg of messages) {
+      // Include full conversation history with tool context
+      for (const msg of messagesRef.current) {
         if (msg.role === 'system') continue;
         apiMessages.push({ role: msg.role, content: msg.content });
+        // Include tool results so the AI has full context
+        if (msg.toolInvocations?.length) {
+          for (const tool of msg.toolInvocations) {
+            if (tool.state === 'result' && tool.result != null) {
+              apiMessages.push({
+                role: 'system',
+                content: `[Tool result from ${tool.toolName}: ${JSON.stringify(tool.result).slice(0, 500)}]`,
+              });
+            }
+          }
+        }
       }
 
-      // New user message
       apiMessages.push({ role: 'user', content: trimmed });
 
       const controller = new AbortController();
       abortRef.current = controller;
 
+      // Reset batching state
+      pendingContentRef.current = '';
+      pendingReasoningRef.current = '';
+      toolInvocationsRef.current = [];
+
       try {
         const response = await fetch(`${apiUrl}/v1/chat/completions`, {
           method: 'POST',
@@ -130,23 +236,15 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
         if (!response.body || typeof response.body.getReader !== 'function') {
           const json = await response.json();
           const content = json.choices?.[0]?.message?.content ?? '';
-          setMessages((prev) => {
-            const updated = [...prev];
-            const last = updated[updated.length - 1];
-            if (last?.role === 'assistant') {
-              updated[updated.length - 1] = { ...last, content };
-            }
-            return updated;
-          });
+          updateAssistant(() => ({ content }));
           return;
         }
 
-        // Stream SSE
+        // Stream SSE (supports named events: event: X\ndata: Y)
         const reader = response.body.getReader();
         const decoder = new TextDecoder();
         let buffer = '';
-        let accumulatedContent = '';
-        const toolInvocations: ToolInvocation[] = [];
+        let currentEventType = '';
 
         try {
           while (true) {
@@ -158,72 +256,151 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
             buffer = lines.pop() ?? '';
 
             for (const line of lines) {
+              // Track named SSE event type
+              if (line.startsWith('event: ')) {
+                currentEventType = line.slice(7).trim();
+                continue;
+              }
+
+              // Reset event type on empty line (SSE event boundary)
+              if (line === '') {
+                currentEventType = '';
+                continue;
+              }
+
               const trimmedLine = line.trim();
-              if (!trimmedLine || !trimmedLine.startsWith('data: ')) continue;
+              if (!trimmedLine.startsWith('data: ')) continue;
               const data = trimmedLine.slice(6);
-              if (data === '[DONE]') continue;
+              if (data === '[DONE]') { currentEventType = ''; continue; }
 
               try {
                 const parsed = JSON.parse(data);
 
-                // Standard content delta
-                const delta = parsed.choices?.[0]?.delta?.content;
-                if (delta) {
-                  accumulatedContent += delta;
-                  setMessages((prev) => {
-                    const updated = [...prev];
-                    const last = updated[updated.length - 1];
-                    if (last?.role === 'assistant') {
-                      updated[updated.length - 1] = {
-                        ...last,
-                        content: accumulatedContent,
-                        toolInvocations: toolInvocations.length > 0 ? [...toolInvocations] : undefined,
-                      };
+                // ── Named SSE events (Alia extensions) ──
+                if (currentEventType) {
+                  switch (currentEventType) {
+                    case 'alia.reasoning': {
+                      const content = parsed.content;
+                      if (content) {
+                        pendingReasoningRef.current += content;
+                        scheduleFlush();
+                      }
+                      currentEventType = '';
+                      continue;
+                    }
+                    case 'alia.tool_result': {
+                      const { tool_call_id, name, output } = parsed;
+                      if (tool_call_id) {
+                        applyToolResult(tool_call_id, name, output);
+                      }
+                      currentEventType = '';
+                      continue;
                     }
-                    return updated;
-                  });
+                    case 'alia.research_progress': {
+                      updateAssistant((last) => ({
+                        researchProgress: {
+                          phase: parsed.phase,
+                          message: parsed.message,
+                          subQuestions: parsed.subQuestions || last.researchProgress?.subQuestions,
+                          sourcesFound: parsed.sourcesFound,
+                          currentQuery: parsed.currentQuery,
+                          iteration: parsed.iteration,
+                        },
+                      }));
+                      currentEventType = '';
+                      continue;
+                    }
+                    case 'alia.plan_preview': {
+                      updateAssistant(() => ({
+                        pendingPlan: {
+                          planId: parsed.planId,
+                          steps: parsed.steps || [],
+                          approved: false,
+                          rejected: false,
+                        },
+                      }));
+                      currentEventType = '';
+                      continue;
+                    }
+                    default:
+                      // Unknown named event — skip
+                      currentEventType = '';
+                      continue;
+                  }
+                }
+
+                // ── Standard OpenAI data events ──
+
+                const choice = parsed.choices?.[0];
+                if (!choice) {
+                  // Legacy alia.tool_call event (type-based, not named SSE)
+                  if (parsed.type === 'alia.tool_call') {
+                    toolInvocationsRef.current = [
+                      ...toolInvocationsRef.current,
+                      {
+                        toolName: parsed.tool || 'unknown',
+                        state: 'call',
+                        args: parsed.args,
+                      },
+                    ];
+                    flushPendingUpdates();
+                  }
+                  // Legacy format matches by toolName (no toolCallId), so can't use applyToolResult
+                  if (parsed.type === 'alia.tool_result') {
+                    toolInvocationsRef.current = toolInvocationsRef.current.map((t) =>
+                      t.toolName === parsed.tool && t.state === 'call'
+                        ? { ...t, state: 'result' as const, result: parsed.result }
+                        : t,
+                    );
+                    flushPendingUpdates();
+                  }
+                  continue;
+                }
+
+                const delta = choice.delta;
+                if (!delta) continue;
+
+                // Reasoning/thinking content (batched)
+                if (delta.reasoning) {
+                  pendingReasoningRef.current += delta.reasoning;
+                  scheduleFlush();
                 }
 
-                // Alia tool call event
-                if (parsed.type === 'alia.tool_call') {
-                  toolInvocations.push({
-                    toolName: parsed.tool || 'unknown',
-                    state: 'call',
-                    args: parsed.args,
-                  });
-                  setMessages((prev) => {
-                    const updated = [...prev];
-                    const last = updated[updated.length - 1];
-                    if (last?.role === 'assistant') {
-                      updated[updated.length - 1] = {
-                        ...last,
-                        toolInvocations: [...toolInvocations],
-                      };
+                // Text content (batched)
+                if (delta.content) {
+                  pendingContentRef.current += delta.content;
+                  scheduleFlush();
+                }
+
+                // Tool calls (OpenAI format: delta.tool_calls)
+                if (delta.tool_calls && Array.isArray(delta.tool_calls)) {
+                  for (const tc of delta.tool_calls) {
+                    const toolCallId = tc.id;
+                    const toolName = tc.function?.name;
+                    if (!toolCallId || !toolName) continue;
+
+                    let args: any;
+                    if (tc.function?.arguments) {
+                      try { args = JSON.parse(tc.function.arguments); } catch { args = { _raw: tc.function.arguments }; }
                     }
-                    return updated;
-                  });
+
+                    const invocation: ToolInvocation = { toolCallId, toolName, state: 'call', args };
+                    const idx = toolInvocationsRef.current.findIndex((t) => t.toolCallId === toolCallId);
+                    if (idx >= 0) {
+                      toolInvocationsRef.current[idx] = invocation;
+                    } else {
+                      toolInvocationsRef.current = [...toolInvocationsRef.current, invocation];
+                    }
+                    flushPendingUpdates();
+                  }
                 }
 
-                // Alia tool result event
-                if (parsed.type === 'alia.tool_result') {
-                  const existing = toolInvocations.find(
-                    (t) => t.toolName === parsed.tool && t.state === 'call',
-                  );
-                  if (existing) {
-                    existing.state = 'result';
-                    existing.result = parsed.result;
+                // Tool results (delta.tool_result)
+                if (delta.tool_result) {
+                  const { tool_call_id, name, output } = delta.tool_result;
+                  if (tool_call_id) {
+                    applyToolResult(tool_call_id, name, output);
                   }
-                  setMessages((prev) => {
-                    const updated = [...prev];
-                    const last = updated[updated.length - 1];
-                    if (last?.role === 'assistant') {
-                      updated[updated.length - 1] = {
-                        ...last,
-                        toolInvocations: [...toolInvocations],
-                      };
-                    }
-                    return updated;
-                  });
                 }
               } catch {
                 // Skip malformed chunks
@@ -237,34 +414,44 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
         if (err?.name === 'AbortError') return;
         const errorMessage = err?.message || 'Something went wrong';
         setError(errorMessage);
-        setMessages((prev) => {
-          const updated = [...prev];
-          const last = updated[updated.length - 1];
-          if (last?.role === 'assistant' && !last.content) {
-            updated[updated.length - 1] = {
-              ...last,
-              content: "I'm having trouble connecting right now. Please try again.",
-            };
-          }
-          return updated;
-        });
+        updateAssistant((last) =>
+          !last.content ? { content: "I'm having trouble connecting right now. Please try again." } : {}
+        );
       } finally {
+        // Flush any remaining batched content
+        flushPendingUpdates();
+        if (flushTimerRef.current) {
+          clearTimeout(flushTimerRef.current);
+          flushTimerRef.current = null;
+        }
         setIsStreaming(false);
         abortRef.current = null;
       }
     },
-    [isStreaming, messages, getToken, apiUrl, model, clientContext],
+    [isStreaming, getToken, apiUrl, model, clientContext, scheduleFlush, flushPendingUpdates, updateAssistant, applyToolResult],
   );
 
-  const clear = useCallback(() => {
+  const stop = useCallback(() => {
     if (abortRef.current) {
       abortRef.current.abort();
       abortRef.current = null;
     }
-    setMessages([]);
+    flushPendingUpdates();
+    if (flushTimerRef.current) {
+      clearTimeout(flushTimerRef.current);
+      flushTimerRef.current = null;
+    }
     setIsStreaming(false);
+  }, [flushPendingUpdates]);
+
+  const clear = useCallback(() => {
+    stop();
+    setMessages([]);
     setError(null);
-  }, []);
+    pendingContentRef.current = '';
+    pendingReasoningRef.current = '';
+    toolInvocationsRef.current = [];
+  }, [stop]);
 
-  return { messages, send, isStreaming, clear, error };
+  return { messages, setMessages, send, isStreaming, stop, clear, error };
 }

package/src/hooks/useAudioLevelMonitor.ts

@@ -0,0 +1,142 @@
+/**
+ * Real-time audio level monitoring using Web Audio API.
+ *
+ * Extracts RMS levels from LiveKit audio tracks (local mic + remote agents)
+ * and returns them at ~20fps for driving wave animations.
+ *
+ * Supports multiple remote audio tracks (primary + cohost) by taking
+ * the max RMS level across all subscribed remote tracks.
+ */
+
+import { useState, useEffect, useRef } from 'react';
+import { Room, RoomEvent, Track } from 'livekit-client';
+
+/** Compute RMS (root mean square) amplitude from an AnalyserNode. Returns 0-1. */
+function getRMS(analyser: AnalyserNode, buffer: any): number {
+  analyser.getFloatTimeDomainData(buffer);
+  let sum = 0;
+  for (let i = 0; i < buffer.length; i++) sum += buffer[i] * buffer[i];
+  return Math.min(1, Math.sqrt(sum / buffer.length) * 8);
+}
+
+export function useAudioLevelMonitor(room: Room | null, isConnected: boolean) {
+  const [captureLevel, setCaptureLevel] = useState(0);
+  const [playbackLevel, setPlaybackLevel] = useState(0);
+  const cleanupRef = useRef<(() => void) | null>(null);
+
+  useEffect(() => {
+    // Tear down previous monitors
+    cleanupRef.current?.();
+    cleanupRef.current = null;
+
+    if (!room || !isConnected) {
+      setCaptureLevel(0);
+      setPlaybackLevel(0);
+      return;
+    }
+
+    // Guard: Web Audio API required
+    if (typeof AudioContext === 'undefined') return;
+
+    const audioCtx = new AudioContext();
+    // Resume AudioContext (may be suspended until user gesture on mobile/WebView)
+    if (audioCtx.state === 'suspended') {
+      audioCtx.resume().catch(() => {});
+    }
+
+    let localAnalyser: AnalyserNode | null = null;
+    let localBuffer: Float32Array | null = null;
+    const remoteAnalysers = new Map<string, { analyser: AnalyserNode; buffer: Float32Array }>();
+    let rafId = 0;
+    let disposed = false;
+
+    // ---------- Local mic ----------
+    const localTrack = room.localParticipant?.getTrackPublication(Track.Source.Microphone)?.track;
+    if (localTrack?.mediaStreamTrack) {
+      try {
+        const source = audioCtx.createMediaStreamSource(new MediaStream([localTrack.mediaStreamTrack]));
+        localAnalyser = audioCtx.createAnalyser();
+        localAnalyser.fftSize = 256;
+        source.connect(localAnalyser);
+        localBuffer = new Float32Array(localAnalyser.fftSize);
+      } catch {
+        // MediaStream creation can fail in some environments
+      }
+    }
+
+    // ---------- Remote agent audio (supports multiple tracks) ----------
+    const setupRemoteAnalyser = (mediaStreamTrack: MediaStreamTrack, key: string) => {
+      if (remoteAnalysers.has(key) || disposed) return;
+      try {
+        const source = audioCtx.createMediaStreamSource(new MediaStream([mediaStreamTrack]));
+        const analyser = audioCtx.createAnalyser();
+        analyser.fftSize = 256;
+        source.connect(analyser);
+        const buffer = new Float32Array(analyser.fftSize);
+        remoteAnalysers.set(key, { analyser, buffer });
+      } catch {
+        // Ignore errors
+      }
+    };
+
+    // Check already-subscribed remote tracks
+    for (const [, p] of room.remoteParticipants) {
+      for (const [, pub] of p.audioTrackPublications) {
+        if (pub.track?.mediaStreamTrack) {
+          setupRemoteAnalyser(pub.track.mediaStreamTrack, `${p.identity}-${pub.trackSid}`);
+        }
+      }
+    }
+
+    // Listen for new remote audio tracks
+    const onTrackSubscribed = (track: any, publication: any, participant: any) => {
+      if (track.kind === Track.Kind.Audio && track.mediaStreamTrack) {
+        setupRemoteAnalyser(track.mediaStreamTrack, `${participant.identity}-${publication.trackSid}`);
+      }
+    };
+    room.on(RoomEvent.TrackSubscribed, onTrackSubscribed);
+
+    // Clean up stale analysers when tracks are unsubscribed
+    const onTrackUnsubscribed = (track: any, publication: any, participant: any) => {
+      const key = `${participant.identity}-${publication.trackSid}`;
+      remoteAnalysers.delete(key);
+    };
+    room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
+
+    // ---------- Poll loop (~20fps) ----------
+    let lastUpdate = 0;
+    const poll = (time: number) => {
+      if (disposed) return;
+      if (time - lastUpdate > 50) {
+        lastUpdate = time;
+        if (localAnalyser && localBuffer) {
+          setCaptureLevel(getRMS(localAnalyser, localBuffer));
+        }
+        // Take max RMS across all remote tracks (primary + cohost)
+        let maxRemoteLevel = 0;
+        for (const [, { analyser, buffer }] of remoteAnalysers) {
+          const level = getRMS(analyser, buffer);
+          if (level > maxRemoteLevel) maxRemoteLevel = level;
+        }
+        setPlaybackLevel(maxRemoteLevel);
+      }
+      rafId = requestAnimationFrame(poll);
+    };
+    rafId = requestAnimationFrame(poll);
+
+    // ---------- Cleanup ----------
+    const dispose = () => {
+      disposed = true;
+      cancelAnimationFrame(rafId);
+      room.off(RoomEvent.TrackSubscribed, onTrackSubscribed);
+      room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
+      remoteAnalysers.clear();
+      audioCtx.close().catch(() => {});
+    };
+    cleanupRef.current = dispose;
+
+    return dispose;
+  }, [room, isConnected]);
+
+  return { captureLevel, playbackLevel };
+}

package/src/hooks/useAudioLevels.ts

@@ -0,0 +1,51 @@
+import { useEffect } from 'react';
+import { useSharedValue, withTiming, Easing } from 'react-native-reanimated';
+import type { AgentState } from '../types';
+
+interface UseAudioLevelsOptions {
+  captureLevel: number;
+  playbackLevel: number;
+  agentState: AgentState;
+  isConnected: boolean;
+}
+
+export function useAudioLevels({
+  captureLevel,
+  playbackLevel,
+  agentState,
+  isConnected,
+}: UseAudioLevelsOptions) {
+  const waveAmplitude = useSharedValue(0);
+
+  useEffect(() => {
+    if (!isConnected) {
+      waveAmplitude.value = withTiming(0, { duration: 300 });
+      return;
+    }
+
+    const amplifiedPlayback = Math.min(1, playbackLevel * 3);
+
+    let target = 0;
+    if (agentState === 'thinking') {
+      target = Math.max(0.15, captureLevel);
+    } else if (agentState === 'speaking') {
+      // 0.4 baseline ensures visible waves while AI speaks
+      target = Math.max(0.4, amplifiedPlayback, captureLevel * 0.5);
+    } else if (agentState === 'listening') {
+      // Subtle pulse while listening; mic level takes over when available
+      target = Math.max(0.08, captureLevel, amplifiedPlayback * 0.5);
+    } else {
+      // Idle — calm, no minimum
+      target = Math.max(captureLevel, amplifiedPlayback * 0.5);
+    }
+
+    // Fast attack, slower decay for natural VU-meter feel
+    const duration = target > waveAmplitude.value ? 60 : 200;
+    waveAmplitude.value = withTiming(target, {
+      duration,
+      easing: Easing.bezier(0.33, 1, 0.68, 1),
+    });
+  }, [captureLevel, playbackLevel, agentState, isConnected]);
+
+  return { waveAmplitude };
+}