bloby-bot 0.17.2

package/supervisor/chat/src/hooks/useBlobyChat.ts

@@ -0,0 +1,266 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import type { WsClient } from '../lib/ws-client';
+import type { ChatMessage, ToolActivity, Attachment, StoredAttachment } from './useChat';
+import { authFetch } from '../lib/auth';
+
+/**
+ * Chat hook for the standalone Bloby chat app.
+ * Loads/persists messages via the DB (worker API).
+ * Supports cross-device sync via chat:sync WS events.
+ */
+export function useBlobyChat(ws: WsClient | null, triggerReload?: number, enabled = true) {
+  const [messages, setMessages] = useState<ChatMessage[]>([]);
+  const [conversationId, setConversationId] = useState<string | null>(null);
+  const [streaming, setStreaming] = useState(false);
+  const [streamBuffer, setStreamBuffer] = useState('');
+  const [tools, setTools] = useState<ToolActivity[]>([]);
+  const [hasMore, setHasMore] = useState(false);
+  const loaded = useRef(false);
+  const conversationIdRef = useRef<string | null>(null);
+  const streamingRef = useRef(false);
+  const loadingOlder = useRef(false);
+
+  // Keep refs in sync with state
+  useEffect(() => { conversationIdRef.current = conversationId; }, [conversationId]);
+  useEffect(() => { streamingRef.current = streaming; }, [streaming]);
+
+  // Parse a raw DB message into a ChatMessage
+  const parseMessage = useCallback((m: any): ChatMessage => {
+    let audioData: string | undefined;
+    if (m.audio_data) {
+      if (m.audio_data.startsWith('data:')) {
+        audioData = m.audio_data;
+      } else if (m.audio_data.includes('/')) {
+        audioData = `/api/files/${m.audio_data}`;
+      } else {
+        audioData = `data:audio/webm;base64,${m.audio_data}`;
+      }
+    }
+
+    let attachments: StoredAttachment[] | undefined;
+    if (m.attachments) {
+      try { attachments = JSON.parse(m.attachments); } catch { /* ignore */ }
+    }
+
+    return {
+      id: m.id,
+      role: m.role,
+      content: m.content,
+      timestamp: m.created_at,
+      audioData,
+      hasAttachments: !!(attachments && attachments.length > 0),
+      attachments,
+    };
+  }, []);
+
+  // Load current conversation from DB (last 20 messages)
+  const loadFromDb = useCallback(async () => {
+    try {
+      const ctx = await authFetch('/api/context/current').then((r) => r.json());
+      if (!ctx.conversationId) return;
+      setConversationId(ctx.conversationId);
+
+      const limit = 20;
+      const res = await authFetch(`/api/conversations/${ctx.conversationId}/messages?limit=${limit}`);
+      if (!res.ok) return;
+      const data = await res.json();
+      if (!data?.length) return;
+
+      const filtered = data.filter((m: any) => m.role === 'user' || m.role === 'assistant');
+      setMessages(filtered.map(parseMessage));
+      setHasMore(data.length >= limit);
+    } catch { /* worker not ready yet */ }
+  }, [parseMessage]);
+
+  // Load older messages (cursor-based pagination)
+  const loadOlder = useCallback(async () => {
+    if (loadingOlder.current || !conversationIdRef.current) return;
+    loadingOlder.current = true;
+    try {
+      const oldestId = messages[0]?.id;
+      if (!oldestId) return;
+      const limit = 20;
+      const res = await authFetch(`/api/conversations/${conversationIdRef.current}/messages?before=${oldestId}&limit=${limit}`);
+      if (!res.ok) return;
+      const data = await res.json();
+      if (!data?.length) { setHasMore(false); return; }
+
+      const filtered = data
+        .filter((m: any) => m.role === 'user' || m.role === 'assistant')
+        .map(parseMessage);
+      setMessages((prev) => [...filtered, ...prev]);
+      if (data.length < limit) setHasMore(false);
+    } catch { /* ignore */ } finally {
+      loadingOlder.current = false;
+    }
+  }, [messages, parseMessage]);
+
+  // Load on mount (only when enabled/authenticated)
+  useEffect(() => {
+    if (!enabled || loaded.current) return;
+    loaded.current = true;
+    loadFromDb();
+  }, [enabled, loadFromDb]);
+
+  // Reload on reconnect (triggerReload changes)
+  useEffect(() => {
+    if (enabled && triggerReload && triggerReload > 0) {
+      loadFromDb();
+    }
+  }, [enabled, triggerReload, loadFromDb]);
+
+  // Periodic DB sync after reconnect while streaming — catches the final bot:response
+  useEffect(() => {
+    if (!enabled || !triggerReload || triggerReload === 0 || !streaming) return;
+    const interval = setInterval(() => {
+      if (!streamingRef.current) {
+        clearInterval(interval);
+        return;
+      }
+      loadFromDb();
+    }, 3000);
+    return () => clearInterval(interval);
+  }, [enabled, triggerReload, streaming, loadFromDb]);
+
+  useEffect(() => {
+    if (!ws) return;
+
+    const unsubs = [
+      ws.on('bot:typing', () => {
+        setStreaming(true);
+        setTools([]);
+      }),
+      ws.on('bot:token', (data: { token: string }) => {
+        setStreamBuffer((buf) => buf + data.token);
+      }),
+      ws.on('bot:tool', (data: { name: string; status?: string }) => {
+        setTools((prev) => {
+          const existing = prev.find((t) => t.name === data.name && t.status === 'running');
+          if (existing) return prev;
+          return [...prev, { name: data.name, status: 'running' }];
+        });
+      }),
+      ws.on('bot:response', (data: { conversationId: string; messageId?: string; content: string }) => {
+        setConversationId(data.conversationId);
+        setMessages((msgs) => [
+          ...msgs,
+          {
+            id: data.messageId || Date.now().toString(),
+            role: 'assistant',
+            content: data.content,
+            timestamp: new Date().toISOString(),
+          },
+        ]);
+        setStreamBuffer('');
+        setStreaming(false);
+        setTools([]);
+      }),
+      ws.on('bot:error', (data: { error: string }) => {
+        setStreamBuffer('');
+        setStreaming(false);
+        setTools([]);
+        setMessages((msgs) => [
+          ...msgs,
+          {
+            id: Date.now().toString(),
+            role: 'assistant',
+            content: `Error: ${data.error}`,
+            timestamp: new Date().toISOString(),
+          },
+        ]);
+      }),
+      // Cross-device sync: append message from another client
+      ws.on('chat:sync', (data: { conversationId: string; message: { role: string; content: string; timestamp: string } }) => {
+        if (conversationIdRef.current && data.conversationId !== conversationIdRef.current) return;
+        setMessages((msgs) => [
+          ...msgs,
+          {
+            id: Date.now().toString(),
+            role: data.message.role as 'user' | 'assistant',
+            content: data.message.content,
+            timestamp: data.message.timestamp,
+          },
+        ]);
+      }),
+      // Server created a new conversation
+      ws.on('chat:conversation-created', (data: { conversationId: string }) => {
+        setConversationId(data.conversationId);
+      }),
+      // Server sends current streaming state on (re)connect
+      ws.on('chat:state', (data: { streaming: boolean; conversationId?: string; buffer?: string }) => {
+        if (data.streaming) {
+          setStreaming(true);
+          if (data.conversationId) setConversationId(data.conversationId);
+          if (data.buffer) setStreamBuffer(data.buffer);
+        }
+      }),
+      // Context cleared (from any client)
+      ws.on('chat:cleared', () => {
+        setMessages([]);
+        setConversationId(null);
+        setStreamBuffer('');
+        setStreaming(false);
+        setTools([]);
+        loaded.current = false;
+      }),
+    ];
+
+    return () => unsubs.forEach((u) => u());
+  }, [ws]);
+
+  const sendMessage = useCallback(
+    (content: string, attachments?: Attachment[], audioData?: string) => {
+      if (!ws || (!content.trim() && (!attachments || attachments.length === 0))) return;
+
+      const userMsg: ChatMessage = {
+        id: Date.now().toString(),
+        role: 'user',
+        content,
+        timestamp: new Date().toISOString(),
+        hasAttachments: !!(attachments && attachments.length > 0),
+        audioData: audioData ? (audioData.startsWith('data:') ? audioData : `data:audio/webm;base64,${audioData}`) : undefined,
+      };
+      setMessages((msgs) => [...msgs, userMsg]);
+
+      const payload: any = { conversationId, content };
+      if (audioData) {
+        payload.audioData = audioData.includes(',') ? audioData.split(',')[1] : audioData;
+      }
+      if (attachments?.length) {
+        payload.attachments = attachments.map((att) => {
+          const match = att.preview.match(/^data:([^;]+);base64,(.+)$/);
+          return {
+            type: att.type,
+            name: att.name,
+            mediaType: match?.[1] || 'application/octet-stream',
+            data: match?.[2] || '',
+          };
+        });
+      }
+      ws.send('user:message', payload);
+    },
+    [ws, conversationId],
+  );
+
+  const stopStreaming = useCallback(() => {
+    if (!ws) return;
+    ws.send('user:stop', { conversationId });
+    setStreaming(false);
+    setStreamBuffer('');
+    setTools([]);
+  }, [ws, conversationId]);
+
+  const clearContext = useCallback(() => {
+    // Send clear to server (which broadcasts to all clients + clears Agent SDK session)
+    if (ws) ws.send('user:clear-context', {});
+    // Optimistic local clear
+    setMessages([]);
+    setConversationId(null);
+    setStreamBuffer('');
+    setStreaming(false);
+    setTools([]);
+    loaded.current = false;
+  }, [ws]);
+
+  return { messages, streaming, streamBuffer, conversationId, tools, hasMore, loadOlder, sendMessage, stopStreaming, clearContext };
+}

package/supervisor/chat/src/hooks/useChat.ts

@@ -0,0 +1,237 @@
+import { useCallback, useEffect, useState, useRef } from 'react';
+import type { WsClient } from '../lib/ws-client';
+
+export interface StoredAttachment {
+  type: string;
+  name: string;
+  mediaType: string;
+  filePath: string;
+}
+
+export interface ChatMessage {
+  id: string;
+  role: 'user' | 'assistant';
+  content: string;
+  timestamp: string; // ISO string
+  hasAttachments?: boolean;
+  audioData?: string; // data URL or HTTP URL for voice messages
+  attachments?: StoredAttachment[];
+}
+
+export interface ToolActivity {
+  name: string;
+  status: 'running' | 'done';
+}
+
+export interface Attachment {
+  id: string;
+  type: 'image' | 'file';
+  name: string;
+  preview: string; // base64 data URL
+}
+
+export function useChat(ws: WsClient | null) {
+  const [messages, setMessages] = useState<ChatMessage[]>([]);
+  const [conversationId, setConversationId] = useState<string | null>(null);
+  const [streaming, setStreaming] = useState(false);
+  const [streamBuffer, setStreamBuffer] = useState('');
+  const [tools, setTools] = useState<ToolActivity[]>([]);
+  const loaded = useRef(false);
+
+  // Load current conversation from DB on mount
+  useEffect(() => {
+    if (loaded.current) return;
+    loaded.current = true;
+    fetch('/api/context/current')
+      .then((r) => r.json())
+      .then((data) => {
+        if (data.conversationId) {
+          setConversationId(data.conversationId);
+        }
+      })
+      .catch(() => {});
+  }, []);
+
+  // Load messages when conversationId is set
+  useEffect(() => {
+    if (!conversationId) return;
+    fetch(`/api/conversations/${conversationId}`)
+      .then((r) => {
+        if (!r.ok) throw new Error('not found');
+        return r.json();
+      })
+      .then((data) => {
+        if (data.messages?.length) {
+          setMessages(
+            data.messages
+              .filter((m: any) => m.role === 'user' || m.role === 'assistant')
+              .map((m: any) => {
+                // Backward compat for audio_data: file path → URL, data: prefix → legacy, else → prepend data URL
+                let audioData: string | undefined;
+                if (m.audio_data) {
+                  if (m.audio_data.startsWith('data:')) {
+                    audioData = m.audio_data; // legacy data URL
+                  } else if (m.audio_data.includes('/')) {
+                    audioData = `/api/files/${m.audio_data}`; // file path → HTTP URL
+                  } else {
+                    audioData = `data:audio/webm;base64,${m.audio_data}`; // raw base64
+                  }
+                }
+
+                // Parse stored attachments
+                let attachments: StoredAttachment[] | undefined;
+                if (m.attachments) {
+                  try {
+                    attachments = JSON.parse(m.attachments);
+                  } catch { /* ignore malformed */ }
+                }
+
+                return {
+                  id: m.id,
+                  role: m.role,
+                  content: m.content,
+                  timestamp: m.created_at,
+                  audioData,
+                  hasAttachments: !!(attachments && attachments.length > 0),
+                  attachments,
+                };
+              }),
+          );
+        }
+      })
+      .catch(() => {
+        // Conversation gone — clear
+        setConversationId(null);
+        fetch('/api/context/clear', { method: 'POST' }).catch(() => {});
+      });
+  }, [conversationId]);
+
+  // Persist conversationId to DB when it changes
+  const prevConvId = useRef<string | null>(null);
+  useEffect(() => {
+    if (conversationId && conversationId !== prevConvId.current) {
+      prevConvId.current = conversationId;
+      fetch('/api/context/set', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ conversationId }),
+      }).catch(() => {});
+    }
+  }, [conversationId]);
+
+  useEffect(() => {
+    if (!ws) return;
+
+    const unsubs = [
+      ws.on('bot:typing', () => {
+        setStreaming(true);
+        setTools([]);
+      }),
+      ws.on('bot:token', (data: { token: string }) => {
+        setStreamBuffer((buf) => buf + data.token);
+      }),
+      ws.on('bot:tool', (data: { name: string; input?: any; status?: string }) => {
+        setTools((prev) => {
+          const existing = prev.find((t) => t.name === data.name && t.status === 'running');
+          if (existing) return prev;
+          return [...prev, { name: data.name, status: 'running' }];
+        });
+      }),
+      ws.on('bot:response', (data: { conversationId: string; messageId?: string; content: string }) => {
+        setConversationId(data.conversationId);
+        setMessages((msgs) => [
+          ...msgs,
+          {
+            id: data.messageId || Date.now().toString(),
+            role: 'assistant',
+            content: data.content,
+            timestamp: new Date().toISOString(),
+          },
+        ]);
+        setStreamBuffer('');
+        setStreaming(false);
+        setTools([]);
+      }),
+      ws.on('bot:error', (data: { error: string }) => {
+        setStreamBuffer('');
+        setStreaming(false);
+        setTools([]);
+        setMessages((msgs) => [
+          ...msgs,
+          {
+            id: Date.now().toString(),
+            role: 'assistant',
+            content: `Error: ${data.error}`,
+            timestamp: new Date().toISOString(),
+          },
+        ]);
+      }),
+    ];
+
+    return () => unsubs.forEach((u) => u());
+  }, [ws]);
+
+  const sendMessage = useCallback(
+    (content: string, attachments?: Attachment[], audioData?: string) => {
+      if (!ws || (!content.trim() && (!attachments || attachments.length === 0))) return;
+
+      // Build optimistic stored attachments from client previews
+      const optimisticAttachments: StoredAttachment[] | undefined = attachments?.map((att) => {
+        const match = att.preview.match(/^data:([^;]+);/);
+        return { type: att.type, name: att.name, mediaType: match?.[1] || 'application/octet-stream', filePath: att.preview };
+      });
+
+      const userMsg: ChatMessage = {
+        id: Date.now().toString(),
+        role: 'user',
+        content,
+        timestamp: new Date().toISOString(),
+        hasAttachments: !!(attachments && attachments.length > 0),
+        audioData: audioData ? (audioData.startsWith('data:') ? audioData : `data:audio/webm;base64,${audioData}`) : undefined,
+        attachments: optimisticAttachments,
+      };
+      setMessages((msgs) => [...msgs, userMsg]);
+
+      // Build WS payload
+      const payload: any = { conversationId, content };
+      if (audioData) {
+        // Send raw base64 (strip data URL prefix)
+        payload.audioData = audioData.includes(',') ? audioData.split(',')[1] : audioData;
+      }
+      if (attachments?.length) {
+        payload.attachments = attachments.map((att) => {
+          const match = att.preview.match(/^data:([^;]+);base64,(.+)$/);
+          return {
+            type: att.type,
+            name: att.name,
+            mediaType: match?.[1] || 'application/octet-stream',
+            data: match?.[2] || '',
+          };
+        });
+      }
+      ws.send('user:message', payload);
+    },
+    [ws, conversationId],
+  );
+
+  const stopStreaming = useCallback(() => {
+    if (!ws || !conversationId) return;
+    ws.send('user:stop', { conversationId });
+    setStreaming(false);
+    setStreamBuffer('');
+    setTools([]);
+  }, [ws, conversationId]);
+
+  const clearContext = useCallback(() => {
+    setMessages([]);
+    setConversationId(null);
+    setStreamBuffer('');
+    setStreaming(false);
+    setTools([]);
+    prevConvId.current = null;
+    loaded.current = false;
+    fetch('/api/context/clear', { method: 'POST' }).catch(() => {});
+  }, []);
+
+  return { messages, streaming, streamBuffer, conversationId, tools, sendMessage, stopStreaming, clearContext };
+}

package/supervisor/chat/src/hooks/useSpeechRecognition.ts

@@ -0,0 +1,137 @@
+import { useRef, useCallback, useMemo } from 'react';
+
+// Extend Window for vendor-prefixed SpeechRecognition
+interface SpeechRecognitionEvent extends Event {
+  results: SpeechRecognitionResultList;
+  resultIndex: number;
+}
+
+type SpeechRecognitionConstructor = new () => SpeechRecognitionInstance;
+
+interface SpeechRecognitionInstance extends EventTarget {
+  continuous: boolean;
+  interimResults: boolean;
+  lang: string;
+  start(): void;
+  stop(): void;
+  abort(): void;
+  onresult: ((event: SpeechRecognitionEvent) => void) | null;
+  onend: (() => void) | null;
+  onerror: ((event: Event & { error: string }) => void) | null;
+}
+
+const getSpeechRecognitionCtor = (): SpeechRecognitionConstructor | null => {
+  if (typeof window === 'undefined') return null;
+  return (
+    (window as unknown as Record<string, SpeechRecognitionConstructor>).SpeechRecognition ??
+    (window as unknown as Record<string, SpeechRecognitionConstructor>).webkitSpeechRecognition ??
+    null
+  );
+};
+
+export const isWebSpeechSupported = getSpeechRecognitionCtor() !== null;
+
+export function useSpeechRecognition() {
+  const instanceRef = useRef<SpeechRecognitionInstance | null>(null);
+  const transcriptRef = useRef('');
+  const resolveRef = useRef<((text: string) => void) | null>(null);
+
+  const isSupported = useMemo(() => isWebSpeechSupported, []);
+
+  const start = useCallback(() => {
+    const Ctor = getSpeechRecognitionCtor();
+    if (!Ctor) {
+      return;
+    }
+
+    // Clean up any lingering instance
+    if (instanceRef.current) {
+      try { instanceRef.current.abort(); } catch {}
+    }
+
+    const recognition = new Ctor();
+    recognition.continuous = false;   // single utterance — matches press-hold-release UX
+    recognition.interimResults = true; // get progressive updates while speaking
+    recognition.lang = navigator.language || 'en-US';
+
+    transcriptRef.current = '';
+    instanceRef.current = recognition;
+
+    recognition.onresult = (event: SpeechRecognitionEvent) => {
+      // Grab the latest result — it's always the most complete/accurate
+      const last = event.results[event.results.length - 1];
+      const text = last[0].transcript;
+      transcriptRef.current = text;
+    };
+
+    recognition.onend = () => {
+      instanceRef.current = null;
+      // If stop() is waiting for a result, resolve it now
+      if (resolveRef.current) {
+        resolveRef.current(transcriptRef.current);
+        resolveRef.current = null;
+      }
+    };
+
+    recognition.onerror = (event) => {
+      console.error('[SpeechRecognition] error:', event.error);
+      instanceRef.current = null;
+      if (resolveRef.current) {
+        resolveRef.current(transcriptRef.current);
+        resolveRef.current = null;
+      }
+    };
+
+    try {
+      recognition.start();
+    } catch (e) {
+      console.error('[SpeechRecognition] start failed:', e);
+    }
+  }, []);
+
+  const stop = useCallback((): Promise<string> => {
+    return new Promise((resolve) => {
+      const instance = instanceRef.current;
+      if (!instance) {
+        // Already ended (e.g. short utterance auto-stopped)
+        resolve(transcriptRef.current);
+        return;
+      }
+
+      // Wait for onend to fire with the final transcript
+      resolveRef.current = resolve;
+
+      try {
+        instance.stop(); // triggers onend after finalizing
+      } catch {
+        instanceRef.current = null;
+        resolve(transcriptRef.current);
+        resolveRef.current = null;
+      }
+
+      // Safety timeout in case onend never fires
+      setTimeout(() => {
+        if (resolveRef.current) {
+          resolveRef.current(transcriptRef.current);
+          resolveRef.current = null;
+          instanceRef.current = null;
+        }
+      }, 2000);
+    });
+  }, []);
+
+  const abort = useCallback(() => {
+    transcriptRef.current = '';
+    const instance = instanceRef.current;
+    if (instance) {
+      try { instance.abort(); } catch {}
+      instanceRef.current = null;
+    }
+    if (resolveRef.current) {
+      resolveRef.current('');
+      resolveRef.current = null;
+    }
+  }, []);
+
+  return { start, stop, abort, isSupported };
+}

package/supervisor/chat/src/lib/auth.ts

@@ -0,0 +1,38 @@
+const TOKEN_KEY = 'bloby_token';
+
+let authFailureCallback: (() => void) | null = null;
+
+export function getAuthToken(): string | null {
+  return localStorage.getItem(TOKEN_KEY);
+}
+
+export function setAuthToken(token: string): void {
+  localStorage.setItem(TOKEN_KEY, token);
+}
+
+export function clearAuthToken(): void {
+  localStorage.removeItem(TOKEN_KEY);
+}
+
+/** Register a callback for when a 401 is received (token expired mid-session) */
+export function onAuthFailure(cb: () => void): void {
+  authFailureCallback = cb;
+}
+
+export async function authFetch(url: string, options: RequestInit = {}): Promise<Response> {
+  const token = getAuthToken();
+  const headers = new Headers(options.headers);
+  if (token) {
+    headers.set('Authorization', `Bearer ${token}`);
+  }
+
+  const res = await fetch(url, { ...options, headers });
+
+  if (res.status === 401 && token) {
+    // Token was present but rejected — it expired
+    clearAuthToken();
+    authFailureCallback?.();
+  }
+
+  return res;
+}