fluxy-bot 0.1.19 → 0.1.21

package/client/src/App.tsx

@@ -37,6 +37,7 @@ export default function App() {
   const [chatOpen, setChatOpen] = useState(false);
   const [showOnboard, setShowOnboard] = useState(false);
   const [botName, setBotName] = useState('Fluxy');
+  const [whisperEnabled, setWhisperEnabled] = useState(false);
   const { ws, connected } = useWebSocket();
   const clearContextRef = useRef<(() => void) | null>(null);
 
@@ -46,16 +47,20 @@ export default function App() {
       .then((s) => {
         if (s.onboard_complete !== 'true') setShowOnboard(true);
         if (s.agent_name) setBotName(s.agent_name);
+        if (s.whisper_enabled === 'true') setWhisperEnabled(true);
       })
       .catch(() => setShowOnboard(true));
   }, []);
 
-  // Refresh bot name after onboarding completes
+  // Refresh bot name + whisper after onboarding completes
   const handleOnboardComplete = () => {
     setShowOnboard(false);
     fetch('/api/settings')
       .then((r) => r.json())
-      .then((s) => { if (s.agent_name) setBotName(s.agent_name); })
+      .then((s) => {
+        if (s.agent_name) setBotName(s.agent_name);
+        setWhisperEnabled(s.whisper_enabled === 'true');
+      })
       .catch(() => {});
   };
 
@@ -104,7 +109,7 @@ export default function App() {
 
           {/* Chat body — fills remaining height */}
           <div className="flex-1 min-h-0 flex flex-col overflow-hidden">
-            <ChatView ws={ws} clearContextRef={clearContextRef} />
+            <ChatView ws={ws} clearContextRef={clearContextRef} whisperEnabled={whisperEnabled} />
           </div>
         </SheetContent>
       </Sheet>

package/client/src/components/Chat/AudioBubble.tsx

@@ -0,0 +1,117 @@
+import { useState, useRef, useCallback, useEffect } from 'react';
+import { Play, Pause } from 'lucide-react';
+
+interface Props {
+  audioData: string; // base64 data URL (data:audio/webm;base64,...)
+}
+
+function formatDuration(s: number): string {
+  if (!isFinite(s) || isNaN(s)) return '0:00';
+  const mins = Math.floor(s / 60);
+  const secs = Math.floor(s % 60);
+  return `${mins}:${secs.toString().padStart(2, '0')}`;
+}
+
+export default function AudioBubble({ audioData }: Props) {
+  const [playing, setPlaying] = useState(false);
+  const [progress, setProgress] = useState(0);
+  const [duration, setDuration] = useState(0);
+  const audioRef = useRef<HTMLAudioElement | null>(null);
+  const barRef = useRef<HTMLDivElement>(null);
+
+  // Create Audio element once
+  useEffect(() => {
+    const audio = new Audio(audioData);
+
+    audio.addEventListener('durationchange', () => {
+      // webm often reports Infinity initially — skip until real
+      if (isFinite(audio.duration)) setDuration(audio.duration);
+    });
+
+    audio.addEventListener('timeupdate', () => {
+      if (isFinite(audio.duration) && audio.duration > 0) {
+        setProgress(audio.currentTime / audio.duration);
+        // Update duration if we didn't get it from durationchange
+        if (duration === 0) setDuration(audio.duration);
+      }
+    });
+
+    audio.addEventListener('ended', () => {
+      setPlaying(false);
+      setProgress(0);
+    });
+
+    // Workaround for webm duration=Infinity: seek to large value to force browser to calculate duration
+    audio.addEventListener('loadedmetadata', () => {
+      if (!isFinite(audio.duration)) {
+        audio.currentTime = 1e10;
+        audio.addEventListener('timeupdate', function seekBack() {
+          if (isFinite(audio.duration)) {
+            setDuration(audio.duration);
+            audio.currentTime = 0;
+            audio.removeEventListener('timeupdate', seekBack);
+          }
+        });
+      }
+    });
+
+    audioRef.current = audio;
+
+    return () => {
+      audio.pause();
+      audio.src = '';
+    };
+  }, [audioData]);
+
+  const togglePlay = useCallback(() => {
+    const audio = audioRef.current;
+    if (!audio) return;
+    if (playing) {
+      audio.pause();
+      setPlaying(false);
+    } else {
+      audio.play();
+      setPlaying(true);
+    }
+  }, [playing]);
+
+  const handleSeek = useCallback((e: React.MouseEvent<HTMLDivElement>) => {
+    const audio = audioRef.current;
+    const bar = barRef.current;
+    if (!audio || !bar || !isFinite(audio.duration)) return;
+    const rect = bar.getBoundingClientRect();
+    const ratio = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width));
+    audio.currentTime = ratio * audio.duration;
+    setProgress(ratio);
+  }, []);
+
+  return (
+    <div className="flex items-center gap-2.5 min-w-[180px]">
+      <button
+        onClick={togglePlay}
+        className="flex items-center justify-center h-8 w-8 shrink-0 rounded-full bg-white/20 hover:bg-white/30 transition-colors"
+      >
+        {playing ? (
+          <Pause className="h-3.5 w-3.5 fill-current" />
+        ) : (
+          <Play className="h-3.5 w-3.5 fill-current ml-0.5" />
+        )}
+      </button>
+      <div className="flex-1 flex flex-col gap-1">
+        <div
+          ref={barRef}
+          onClick={handleSeek}
+          className="h-1 bg-white/20 rounded-full cursor-pointer relative"
+        >
+          <div
+            className="absolute inset-y-0 left-0 bg-white/70 rounded-full transition-[width] duration-100"
+            style={{ width: `${progress * 100}%` }}
+          />
+        </div>
+        <span className="text-[10px] opacity-60 tabular-nums">
+          {formatDuration(playing ? progress * duration : duration)}
+        </span>
+      </div>
+    </div>
+  );
+}

package/client/src/components/Chat/ChatView.tsx

@@ -7,9 +7,10 @@ interface Props {
   ws: WsClient | null;
   onClearContext?: () => void;
   clearContextRef?: React.MutableRefObject<(() => void) | null>;
+  whisperEnabled?: boolean;
 }
 
-export default function ChatView({ ws, clearContextRef }: Props) {
+export default function ChatView({ ws, clearContextRef, whisperEnabled }: Props) {
   const { messages, streaming, streamBuffer, tools, sendMessage, stopStreaming, clearContext } = useChat(ws);
 
   // Expose clearContext to parent via ref
@@ -18,7 +19,7 @@ export default function ChatView({ ws, clearContextRef }: Props) {
   return (
     <div className="flex flex-col h-full overflow-hidden">
       <MessageList messages={messages} streaming={streaming} streamBuffer={streamBuffer} tools={tools} />
-      <InputBar onSend={sendMessage} onStop={stopStreaming} streaming={streaming} />
+      <InputBar onSend={sendMessage} onStop={stopStreaming} streaming={streaming} whisperEnabled={whisperEnabled} />
     </div>
   );
 }

package/client/src/components/Chat/ImageLightbox.tsx

@@ -0,0 +1,86 @@
+import { useCallback, useEffect } from 'react';
+import { motion, AnimatePresence } from 'framer-motion';
+import { ChevronLeft, ChevronRight, X } from 'lucide-react';
+
+interface Props {
+  images: string[];
+  index: number;
+  onClose: () => void;
+  onNavigate: (index: number) => void;
+}
+
+export default function ImageLightbox({ images, index, onClose, onNavigate }: Props) {
+  const goPrev = useCallback(() => {
+    if (index > 0) onNavigate(index - 1);
+  }, [index, onNavigate]);
+
+  const goNext = useCallback(() => {
+    if (index < images.length - 1) onNavigate(index + 1);
+  }, [index, images.length, onNavigate]);
+
+  useEffect(() => {
+    const handleKey = (e: KeyboardEvent) => {
+      if (e.key === 'Escape') onClose();
+      if (e.key === 'ArrowLeft') goPrev();
+      if (e.key === 'ArrowRight') goNext();
+    };
+    window.addEventListener('keydown', handleKey);
+    return () => window.removeEventListener('keydown', handleKey);
+  }, [onClose, goPrev, goNext]);
+
+  return (
+    <AnimatePresence>
+      <motion.div
+        initial={{ opacity: 0 }}
+        animate={{ opacity: 1 }}
+        exit={{ opacity: 0 }}
+        transition={{ duration: 0.15 }}
+        className="fixed inset-0 z-50 flex items-center justify-center bg-black/90"
+        onClick={onClose}
+      >
+        {/* Close button */}
+        <button
+          onClick={onClose}
+          className="absolute top-4 right-4 z-10 p-2 rounded-full bg-white/10 hover:bg-white/20 transition-colors text-white"
+        >
+          <X className="h-5 w-5" />
+        </button>
+
+        {/* Image counter */}
+        {images.length > 1 && (
+          <div className="absolute top-4 left-1/2 -translate-x-1/2 text-white/70 text-sm tabular-nums">
+            {index + 1} / {images.length}
+          </div>
+        )}
+
+        {/* Left arrow */}
+        {index > 0 && (
+          <button
+            onClick={(e) => { e.stopPropagation(); goPrev(); }}
+            className="absolute left-3 top-1/2 -translate-y-1/2 z-10 p-2 rounded-full bg-white/10 hover:bg-white/20 transition-colors text-white"
+          >
+            <ChevronLeft className="h-6 w-6" />
+          </button>
+        )}
+
+        {/* Right arrow */}
+        {index < images.length - 1 && (
+          <button
+            onClick={(e) => { e.stopPropagation(); goNext(); }}
+            className="absolute right-3 top-1/2 -translate-y-1/2 z-10 p-2 rounded-full bg-white/10 hover:bg-white/20 transition-colors text-white"
+          >
+            <ChevronRight className="h-6 w-6" />
+          </button>
+        )}
+
+        {/* Image */}
+        <img
+          src={images[index]}
+          alt=""
+          className="max-h-[85vh] max-w-[90vw] object-contain rounded-lg"
+          onClick={(e) => e.stopPropagation()}
+        />
+      </motion.div>
+    </AnimatePresence>
+  );
+}

package/client/src/components/Chat/InputBar.tsx

@@ -4,9 +4,10 @@ import { motion, AnimatePresence } from 'framer-motion';
 import type { Attachment } from '../../hooks/useChat';
 
 interface Props {
-  onSend: (msg: string, attachments?: Attachment[]) => void;
+  onSend: (msg: string, attachments?: Attachment[], audioData?: string) => void;
   onStop: () => void;
   streaming: boolean;
+  whisperEnabled?: boolean;
 }
 
 function formatTime(s: number) {
@@ -50,7 +51,7 @@ function compressImage(dataUrl: string, maxBytes = 4 * 1024 * 1024): Promise<str
   });
 }
 
-export default function InputBar({ onSend, onStop, streaming }: Props) {
+export default function InputBar({ onSend, onStop, streaming, whisperEnabled }: Props) {
   const [text, setText] = useState('');
   const [attachments, setAttachments] = useState<Attachment[]>([]);
   const [isRecording, setIsRecording] = useState(false);
@@ -68,6 +69,9 @@ export default function InputBar({ onSend, onStop, streaming }: Props) {
   const holdTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
   const isHolding = useRef(false);
   const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const audioChunksRef = useRef<Blob[]>([]);
+  const streamRef = useRef<MediaStream | null>(null);
 
   // Auto-resize textarea up to 4 lines, then scroll
   useEffect(() => {
@@ -89,14 +93,58 @@ export default function InputBar({ onSend, onStop, streaming }: Props) {
     if (intervalRef.current) clearInterval(intervalRef.current);
     if (holdTimerRef.current) { clearTimeout(holdTimerRef.current); holdTimerRef.current = null; }
     isHolding.current = false;
-    if (!cancelled) {
-      // Future: send recorded audio
+
+    const recorder = mediaRecorderRef.current;
+    const stream = streamRef.current;
+
+    if (cancelled || !recorder || recorder.state === 'inactive') {
+      // Clean up mic
+      stream?.getTracks().forEach((t) => t.stop());
+      mediaRecorderRef.current = null;
+      streamRef.current = null;
+      audioChunksRef.current = [];
+    } else {
+      // Stop recorder — ondataavailable + onstop will fire
+      recorder.onstop = async () => {
+        stream?.getTracks().forEach((t) => t.stop());
+        const blob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
+        audioChunksRef.current = [];
+        mediaRecorderRef.current = null;
+        streamRef.current = null;
+
+        if (blob.size < 1000) return; // too small, skip
+
+        // Convert to base64
+        const reader = new FileReader();
+        reader.onloadend = async () => {
+          const dataUrl = reader.result as string;
+          const base64 = dataUrl.split(',')[1];
+          if (!base64) return;
+
+          try {
+            const res = await fetch('/api/whisper/transcribe', {
+              method: 'POST',
+              headers: { 'Content-Type': 'application/json' },
+              body: JSON.stringify({ audio: base64 }),
+            });
+            const data = await res.json();
+            if (data.transcript?.trim()) {
+              onSend(data.transcript.trim(), undefined, dataUrl);
+            }
+          } catch {
+            // Transcription failed silently
+          }
+        };
+        reader.readAsDataURL(blob);
+      };
+      recorder.stop();
     }
+
     if (micRef.current) micRef.current.style.transform = '';
     setIsRecording(false);
     setRecordingTime(0);
     dragRef.current = 0;
-  }, []);
+  }, [onSend]);
 
   // ── File handling ──
 
@@ -169,10 +217,25 @@ export default function InputBar({ onSend, onStop, streaming }: Props) {
     dragRef.current = 0;
     (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
 
-    holdTimerRef.current = setTimeout(() => {
-      isHolding.current = true;
-      setIsRecording(true);
-      setRecordingTime(0);
+    holdTimerRef.current = setTimeout(async () => {
+      if (!whisperEnabled) return;
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        streamRef.current = stream;
+        const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') ? 'audio/webm;codecs=opus' : 'audio/webm';
+        const recorder = new MediaRecorder(stream, { mimeType });
+        audioChunksRef.current = [];
+        recorder.ondataavailable = (e) => {
+          if (e.data.size > 0) audioChunksRef.current.push(e.data);
+        };
+        mediaRecorderRef.current = recorder;
+        recorder.start();
+        isHolding.current = true;
+        setIsRecording(true);
+        setRecordingTime(0);
+      } catch {
+        // Mic permission denied or not available
+      }
     }, 200);
   }, []);
 
@@ -372,6 +435,7 @@ export default function InputBar({ onSend, onStop, streaming }: Props) {
           </motion.div>
         )}
       </AnimatePresence>
+
     </div>
   );
 }

package/client/src/components/Chat/MessageBubble.tsx

@@ -3,12 +3,17 @@ import remarkGfm from 'remark-gfm';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
 import { Paperclip } from 'lucide-react';
+import AudioBubble from './AudioBubble';
+import type { StoredAttachment } from '../../hooks/useChat';
 
 interface Props {
   role: 'user' | 'assistant';
   content: string;
   timestamp?: string;
   hasAttachments?: boolean;
+  audioData?: string;
+  attachments?: StoredAttachment[];
+  onImageClick?: (images: string[], index: number) => void;
 }
 
 function formatTime(iso: string): string {
@@ -20,15 +25,58 @@ function formatTime(iso: string): string {
   }
 }
 
-export default function MessageBubble({ role, content, timestamp, hasAttachments }: Props) {
+export default function MessageBubble({ role, content, timestamp, hasAttachments, audioData, attachments, onImageClick }: Props) {
   const isUser = role === 'user';
   const time = timestamp ? formatTime(timestamp) : '';
 
+  // Separate image and document attachments
+  const imageAtts = attachments?.filter((a) => a.mediaType?.startsWith('image/')) || [];
+  const docAtts = attachments?.filter((a) => !a.mediaType?.startsWith('image/')) || [];
+
+  // Resolve image URLs
+  const imageUrls = imageAtts.map((a) =>
+    a.filePath.startsWith('data:') ? a.filePath : `/api/files/${a.filePath}`
+  );
+
   if (isUser) {
+    // Voice message: audio-only, no transcript text
+    if (audioData) {
+      return (
+        <div className="flex flex-col items-end gap-0.5">
+          <div className="max-w-[75%] rounded-2xl px-4 py-2.5 text-sm leading-relaxed bg-primary text-primary-foreground">
+            <AudioBubble audioData={audioData} />
+          </div>
+          {time && <span className="text-[10px] text-muted-foreground/50 px-1">{time}</span>}
+        </div>
+      );
+    }
+
     return (
       <div className="flex flex-col items-end gap-0.5">
         <div className="max-w-[75%] rounded-2xl px-4 py-2.5 text-sm leading-relaxed whitespace-pre-wrap bg-primary text-primary-foreground break-words overflow-hidden" style={{ overflowWrap: 'anywhere' }}>
-          {hasAttachments && (
+          {/* Image thumbnails */}
+          {imageUrls.length > 0 && (
+            <div className="flex gap-1.5 flex-wrap mb-2">
+              {imageUrls.map((url, i) => (
+                <img
+                  key={i}
+                  src={url}
+                  alt={imageAtts[i]?.name || 'attachment'}
+                  className="w-28 h-28 rounded-lg object-cover cursor-pointer border border-white/10 hover:opacity-80 transition-opacity"
+                  onClick={() => onImageClick?.(imageUrls, i)}
+                />
+              ))}
+            </div>
+          )}
+          {/* Document attachments */}
+          {docAtts.length > 0 && (
+            <span className="inline-flex items-center gap-1 text-primary-foreground/60 mr-1.5">
+              <Paperclip className="h-3 w-3" />
+              {docAtts.length > 1 && <span className="text-xs">{docAtts.length}</span>}
+            </span>
+          )}
+          {/* Fallback paperclip for legacy messages with no parsed attachments */}
+          {!attachments?.length && hasAttachments && (
             <span className="inline-flex items-center gap-1 text-primary-foreground/60 mr-1.5">
               <Paperclip className="h-3 w-3" />
             </span>

package/client/src/components/Chat/MessageList.tsx

@@ -1,7 +1,8 @@
-import { useEffect, useRef } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
 import type { ChatMessage, ToolActivity } from '../../hooks/useChat';
 import MessageBubble from './MessageBubble';
 import TypingIndicator from './TypingIndicator';
+import ImageLightbox from './ImageLightbox';
 
 interface Props {
   messages: ChatMessage[];
@@ -12,6 +13,11 @@ interface Props {
 
 export default function MessageList({ messages, streaming, streamBuffer, tools }: Props) {
   const bottomRef = useRef<HTMLDivElement>(null);
+  const [lightbox, setLightbox] = useState<{ images: string[]; index: number } | null>(null);
+
+  const handleImageClick = useCallback((images: string[], index: number) => {
+    setLightbox({ images, index });
+  }, []);
 
   useEffect(() => {
     bottomRef.current?.scrollIntoView({ behavior: 'smooth' });
@@ -40,6 +46,9 @@ export default function MessageList({ messages, streaming, streamBuffer, tools }
             content={msg.content}
             timestamp={msg.timestamp}
             hasAttachments={msg.hasAttachments}
+            audioData={msg.audioData}
+            attachments={msg.attachments}
+            onImageClick={handleImageClick}
           />
         ))}
 
@@ -49,6 +58,15 @@ export default function MessageList({ messages, streaming, streamBuffer, tools }
         )}
       </div>
       <div ref={bottomRef} />
+
+      {lightbox && (
+        <ImageLightbox
+          images={lightbox.images}
+          index={lightbox.index}
+          onClose={() => setLightbox(null)}
+          onNavigate={(i) => setLightbox((prev) => prev ? { ...prev, index: i } : null)}
+        />
+      )}
     </div>
   );
 }

package/client/src/components/FluxyFab.tsx

@@ -14,10 +14,14 @@ export default function FluxyFab({ onClick }: Props) {
       role="button"
       aria-label="Open Fluxy chat"
     >
-      <img
-        src="/fluxy.png"
-        alt="Fluxy"
-        className="h-11 w-auto drop-shadow-lg"
+      <video
+        src="/fluxy_tilts.webm"
+        poster="/fluxy_frame1.png"
+        autoPlay
+        loop
+        muted
+        playsInline
+        className="h-11 w-auto drop-shadow-lg pointer-events-none"
         draggable={false}
       />
     </motion.div>

package/client/src/hooks/useChat.ts

@@ -1,12 +1,21 @@
 import { useCallback, useEffect, useState, useRef } from 'react';
 import type { WsClient } from '../lib/ws-client';
 
+export interface StoredAttachment {
+  type: string;
+  name: string;
+  mediaType: string;
+  filePath: string;
+}
+
 export interface ChatMessage {
   id: string;
   role: 'user' | 'assistant';
   content: string;
   timestamp: string; // ISO string
   hasAttachments?: boolean;
+  audioData?: string; // data URL or HTTP URL for voice messages
+  attachments?: StoredAttachment[];
 }
 
 export interface ToolActivity {
@@ -56,12 +65,37 @@ export function useChat(ws: WsClient | null) {
           setMessages(
             data.messages
               .filter((m: any) => m.role === 'user' || m.role === 'assistant')
-              .map((m: any) => ({
-                id: m.id,
-                role: m.role,
-                content: m.content,
-                timestamp: m.created_at,
-              })),
+              .map((m: any) => {
+                // Backward compat for audio_data: file path → URL, data: prefix → legacy, else → prepend data URL
+                let audioData: string | undefined;
+                if (m.audio_data) {
+                  if (m.audio_data.startsWith('data:')) {
+                    audioData = m.audio_data; // legacy data URL
+                  } else if (m.audio_data.includes('/')) {
+                    audioData = `/api/files/${m.audio_data}`; // file path → HTTP URL
+                  } else {
+                    audioData = `data:audio/webm;base64,${m.audio_data}`; // raw base64
+                  }
+                }
+
+                // Parse stored attachments
+                let attachments: StoredAttachment[] | undefined;
+                if (m.attachments) {
+                  try {
+                    attachments = JSON.parse(m.attachments);
+                  } catch { /* ignore malformed */ }
+                }
+
+                return {
+                  id: m.id,
+                  role: m.role,
+                  content: m.content,
+                  timestamp: m.created_at,
+                  audioData,
+                  hasAttachments: !!(attachments && attachments.length > 0),
+                  attachments,
+                };
+              }),
           );
         }
       })
@@ -138,20 +172,32 @@ export function useChat(ws: WsClient | null) {
   }, [ws]);
 
   const sendMessage = useCallback(
-    (content: string, attachments?: Attachment[]) => {
+    (content: string, attachments?: Attachment[], audioData?: string) => {
       if (!ws || (!content.trim() && (!attachments || attachments.length === 0))) return;
 
+      // Build optimistic stored attachments from client previews
+      const optimisticAttachments: StoredAttachment[] | undefined = attachments?.map((att) => {
+        const match = att.preview.match(/^data:([^;]+);/);
+        return { type: att.type, name: att.name, mediaType: match?.[1] || 'application/octet-stream', filePath: att.preview };
+      });
+
       const userMsg: ChatMessage = {
         id: Date.now().toString(),
         role: 'user',
         content,
         timestamp: new Date().toISOString(),
         hasAttachments: !!(attachments && attachments.length > 0),
+        audioData: audioData ? (audioData.startsWith('data:') ? audioData : `data:audio/webm;base64,${audioData}`) : undefined,
+        attachments: optimisticAttachments,
       };
       setMessages((msgs) => [...msgs, userMsg]);
 
       // Build WS payload
       const payload: any = { conversationId, content };
+      if (audioData) {
+        // Send raw base64 (strip data URL prefix)
+        payload.audioData = audioData.includes(',') ? audioData.split(',')[1] : audioData;
+      }
       if (attachments?.length) {
         payload.attachments = attachments.map((att) => {
           const match = att.preview.match(/^data:([^;]+);base64,(.+)$/);