madarx 2.16.3 → 2.18.0

package/app/components/dashboard/voice-panel.jsx

@@ -1,9 +1,23 @@
 "use client";
 
-import { memo, useCallback, useEffect, useMemo, useState } from 'react';
+import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import VoiceOrb from './voice-orb';
 import { useRealtimeVoice } from './use-realtime-voice';
 import { useDashboard } from './dashboard-context';
+import { useAccountSession } from '../account-session.jsx';
+
+// Derive a friendly first name from the signed-in session. Prefers an explicit
+// display name; otherwise capitalises the email local-part (ak@madarx.io → Ak).
+function greetingNameFrom(session) {
+  const dn = session?.display_name && String(session.display_name).trim();
+  if (dn) return dn.split(/\s+/)[0];
+  const email = session?.email && String(session.email);
+  if (email && email.includes('@')) {
+    const local = email.split('@')[0].replace(/[._-]+/g, ' ').trim().split(/\s+/)[0];
+    if (local) return local.charAt(0).toUpperCase() + local.slice(1);
+  }
+  return '';
+}
 
 const MODE_LABELS = {
   realtime: 'Live',
@@ -36,6 +50,7 @@ function VoicePanel() {
     spaces: spacesHook,
     state,
     chat,
+    mainAgent,
     setVoiceOpen,
     setConfigTab,
     setConfigOpen,
@@ -83,8 +98,100 @@ function VoicePanel() {
   const [providerError, setProviderError] = useState(null);
   const [ttsSpeaking, setTtsSpeaking] = useState(false);
   const [collapsed, setCollapsed] = useState(() => load('madarx:voice-collapsed', '0') === '1');
+  // Brain selection. 'realtime' (default) is the proven in-browser-brain path
+  // (Realtime reasons + calls tools + speaks). 'transport' routes voice through
+  // the same main-agent stream as typed chat (Realtime = STT only, TTS = mouth)
+  // — opt-in until verified live, set localStorage madarx:voice-brain='transport'.
+  const [brain] = useState(() => load('madarx:voice-brain', 'realtime'));
+  const transport = brain === 'transport';
+
+  // Speak an arbitrary string via the TTS lane. Generalised from speakLast so
+  // the orchestrator's reply can be voiced in transport mode.
+  const speakText = useCallback(async (text) => {
+    const say = String(text || '').trim();
+    if (!say) return;
+    setProviderError(null);
+    try {
+      setTtsSpeaking(true);
+      const res = await fetch('/api/voice/tts', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          text: say.slice(0, 3800),
+          provider: ttsProvider,
+          voice: ttsVoice || undefined,
+          project_id: projectId,
+        }),
+      });
+      if (!res.ok) throw new Error(`tts ${res.status}`);
+      const blob = await res.blob();
+      const url = URL.createObjectURL(blob);
+      const audio = new Audio(url);
+      audio.onended = () => { setTtsSpeaking(false); URL.revokeObjectURL(url); };
+      audio.onerror = () => { setTtsSpeaking(false); URL.revokeObjectURL(url); };
+      await audio.play();
+    } catch (err) {
+      setTtsSpeaking(false);
+      setProviderError(String(err?.message || err));
+    }
+  }, [projectId, ttsProvider, ttsVoice]);
+
+  // Transport loop: a final user utterance from Realtime STT is forwarded to
+  // the main orchestrator (same brain + tools + gates as typed chat). We mark a
+  // reply pending so the busy→idle effect below can voice the answer.
+  const pendingSpeakRef = useRef(false);
+  const onUserUtterance = useCallback((text) => {
+    const said = String(text || '').trim();
+    if (!said) return;
+    if (mainAgent?.busy) return; // a turn is already in flight; drop barge-in
+    pendingSpeakRef.current = true;
+    mainAgent?.command?.(said);
+  }, [mainAgent]);
 
-  const realtime = useRealtimeVoice({ projectId, agentId, activeSpace, allSpaces });
+  const { session: account } = useAccountSession();
+  const greetingName = useMemo(() => greetingNameFrom(account), [account]);
+  const greetingText = greetingName ? `Hello, ${greetingName}.` : 'Hello.';
+
+  const realtime = useRealtimeVoice({ projectId, agentId, activeSpace, allSpaces, transport, greetingText, onUserUtterance });
+
+  // Transport-mode greeting: the Realtime model is silent, so speak the hello
+  // through the TTS lane once the line goes live. Legacy mode greets via the
+  // model itself (handled in the hook's dc.onopen).
+  const greetedRef = useRef(false);
+  useEffect(() => {
+    if (transport && realtime.state === 'live' && !greetedRef.current) {
+      greetedRef.current = true;
+      speakText(greetingText);
+    }
+    if (realtime.state === 'idle' || realtime.state === 'ending') greetedRef.current = false;
+  }, [transport, realtime.state, greetingText, speakText]);
+
+  // When a voice-initiated orchestrator turn finishes (busy true→false), speak
+  // the freshest assistant reply. Approval pauses surface separately below.
+  const prevBusyRef = useRef(false);
+  const spokenApprovalRef = useRef(false);
+  useEffect(() => {
+    const wasBusy = prevBusyRef.current;
+    prevBusyRef.current = mainAgent?.busy || false;
+    if (!transport) return;
+    if (wasBusy && !mainAgent?.busy && pendingSpeakRef.current) {
+      pendingSpeakRef.current = false;
+      // If the turn paused on an approval, say so instead of reading a
+      // half-finished answer; the Approve/Deny controls live in the chat panel.
+      const awaiting = (mainAgent?.liveSteps || []).some(s => s.status === 'awaiting_approval');
+      if (awaiting) {
+        if (!spokenApprovalRef.current) {
+          spokenApprovalRef.current = true;
+          speakText('That action needs your approval. Open the chat to approve or deny.');
+        }
+        return;
+      }
+      spokenApprovalRef.current = false;
+      for (let i = chat.length - 1; i >= 0; i--) {
+        if (chat[i].role === 'agent' && chat[i].text) { speakText(chat[i].text); break; }
+      }
+    }
+  }, [mainAgent?.busy, mainAgent?.liveSteps, transport, chat, speakText]);
 
   // Capability discovery — lets us hide modes we can't run.
   useEffect(() => {
@@ -122,13 +229,21 @@ function VoicePanel() {
     return () => window.removeEventListener('storage', onStorage);
   }, []);
 
+  const awaitingApproval = useMemo(
+    () => transport && (mainAgent?.liveSteps || []).some(s => s.status === 'awaiting_approval'),
+    [transport, mainAgent?.liveSteps],
+  );
+
   const orbState = useMemo(() => {
     if (realtime.error) return 'error';
-    if (realtime.pendingToolCalls.length > 0) return 'approval';
+    if (awaitingApproval || realtime.pendingToolCalls.length > 0) return 'approval';
+    // Transport mode keeps the WebRTC line live while the orchestrator thinks
+    // and the TTS lane speaks — surface those as an active orb.
+    if (transport && (mainAgent?.busy || ttsSpeaking)) return 'live';
     if (mode === 'realtime') return realtime.state;
     if (ttsSpeaking) return 'live';
     return 'idle';
-  }, [mode, realtime.error, realtime.pendingToolCalls.length, realtime.state, ttsSpeaking]);
+  }, [mode, transport, awaitingApproval, mainAgent?.busy, realtime.error, realtime.pendingToolCalls.length, realtime.state, ttsSpeaking]);
 
   const stop = useCallback(() => {
     realtime.stop();
@@ -142,35 +257,31 @@ function VoicePanel() {
 
   // Speak last assistant message via TTS (used in tts mode).
   const speakLast = useCallback(async () => {
-    setProviderError(null);
     const text = await onSpeakLast?.();
-    if (!text) return;
-    try {
-      setTtsSpeaking(true);
-      const res = await fetch('/api/voice/tts', {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          text: String(text).slice(0, 3800),
-          provider: ttsProvider,
-          voice: ttsVoice || undefined,
-          project_id: projectId,
-        }),
-      });
-      if (!res.ok) throw new Error(`tts ${res.status}`);
-      const blob = await res.blob();
-      const url = URL.createObjectURL(blob);
-      const audio = new Audio(url);
-      audio.onended = () => { setTtsSpeaking(false); URL.revokeObjectURL(url); };
-      audio.onerror = () => { setTtsSpeaking(false); URL.revokeObjectURL(url); };
-      await audio.play();
-    } catch (err) {
-      setTtsSpeaking(false);
-      setProviderError(String(err?.message || err));
-    }
-  }, [onSpeakLast, projectId, ttsProvider, ttsVoice]);
+    await speakText(text);
+  }, [onSpeakLast, speakText]);
 
   const liveTranscript = realtime.transcript.slice(-8);
+
+  // Activity strip source. In transport mode the tools run through the
+  // orchestrator, so surface mainAgent.liveSteps; otherwise use the Realtime
+  // hook's own tool-call tracking.
+  const pendingPills = transport
+    ? (mainAgent?.liveSteps || [])
+        .filter(s => s.status === 'running')
+        .map(s => ({ call_id: s.id, name: s.name }))
+    : realtime.pendingToolCalls;
+  const recentPills = transport
+    ? (mainAgent?.liveSteps || [])
+        .filter(s => s.status !== 'running')
+        .slice(-6)
+        .map(s => ({
+          call_id: s.id,
+          name: s.name,
+          status: s.status === 'awaiting_approval' ? 'approval' : (s.status === 'error' ? 'error' : 'ok'),
+          duration_ms: s.durationMs || 0,
+        }))
+    : (realtime.recentToolCalls || []);
   const caps = providers?.capabilities;
   const realtimeAvailable = Boolean(caps?.realtime?.openai);
   const ttsAvailable = Boolean(caps?.tts?.openai || caps?.tts?.elevenlabs || caps?.tts?.local);
@@ -219,6 +330,9 @@ function VoicePanel() {
     if (realtime.error) return STATE_COPY.error;
     if (realtime.pendingToolCalls.length > 0) return STATE_COPY.approval;
     if (mode === 'realtime') {
+      if (awaitingApproval) return 'Needs approval — open chat';
+      if (transport && mainAgent?.busy) return 'Thinking…';
+      if (transport && ttsSpeaking) return 'Speaking…';
       if (realtime.muted && isLive) return 'Muted — tap orb to end';
       if (isLive) return 'Listening — tap orb to end';
       if (isConnecting) return 'Connecting…';
@@ -302,21 +416,19 @@ function VoicePanel() {
         <div
           className={
             'voice-panel__activity' +
-            (realtime.pendingToolCalls.length > 0 || (realtime.recentToolCalls && realtime.recentToolCalls.length > 0)
-              ? ' is-active'
-              : '')
+            (pendingPills.length > 0 || recentPills.length > 0 ? ' is-active' : '')
           }
           aria-live="polite"
         >
           <div className="voice-panel__activity-track">
-            {realtime.pendingToolCalls.map(c => (
+            {pendingPills.map(c => (
               <span key={c.call_id} className="voice-panel__activity-pill voice-panel__activity-pill--running">
                 <span className="voice-panel__activity-dot" aria-hidden />
                 {c.name.replace(/_/g, ' ')}
               </span>
             ))}
-            {(realtime.recentToolCalls || [])
-              .filter(r => !realtime.pendingToolCalls.some(p => p.call_id === r.call_id))
+            {recentPills
+              .filter(r => !pendingPills.some(p => p.call_id === r.call_id))
               .slice(-6)
               .map(r => (
                 <span

package/app/components/smart-search.jsx

@@ -232,6 +232,17 @@ export default function SmartSearch() {
     return () => window.removeEventListener('madarx:open-setting', onOpenSetting);
   }, [openSetting]);
 
+  // Let chrome surfaces (the persistent top toolbar) open the palette without
+  // synthesizing a keydown. Mirrors the ⌘K toggle.
+  useEffect(() => {
+    function onOpenSearch(event) {
+      const q = event?.detail?.query;
+      openWithQuery(typeof q === 'string' ? q : '');
+    }
+    window.addEventListener('madarx:open-search', onOpenSearch);
+    return () => window.removeEventListener('madarx:open-search', onOpenSearch);
+  }, []);
+
   useEffect(() => {
     function onKeyDown(event) {
       const key = (event.key || '').toLowerCase();

package/app/components/ui/marketplace/install-toggle.jsx

@@ -1,18 +1,25 @@
+'use client';
+
+import { useState } from 'react';
+
 /**
- * Install toggle button — server-action backed via the surrounding `<form>`.
- * Generalised from the .tp-switch toggle on the legacy tools page.
+ * Install toggle button.
  *
- * The form action must be a Next.js server action that calls
- * `setItemInstalled(id, !installed)`. The action lives outside this component
- * so the toggle stays purely presentational and can be reused everywhere a
- * marketplace item is rendered.
+ * Two modes:
+ *  - Client mode (preferred): pass `onToggle(nextInstalled) => Promise`. The
+ *    toggle flips in place via fetch — no full-page navigation. Used by the
+ *    marketplace shell so installing a tool updates the card without a refresh.
+ *  - Server-action mode (fallback): pass `action` (a Next.js server action). The
+ *    surrounding `<form>` submits and the action redirects with a toast. Kept for
+ *    progressive enhancement / no-JS.
  *
  * Hidden fields `current_kind` and `slug_hint` are echoed back to the server
  * action so the post-redirect toast can identify which item changed and which
  * tab to restore.
  *
  * @param {{
- *   action: any,
+ *   action?: any,
+ *   onToggle?: (nextInstalled: boolean) => Promise<any>,
  *   itemId: string,
  *   installed: boolean,
  *   available?: boolean,
@@ -24,6 +31,7 @@
  */
 export default function InstallToggle({
   action,
+  onToggle,
   itemId,
   installed,
   available = true,
@@ -32,8 +40,35 @@ export default function InstallToggle({
   slugHint,
   currentKind,
 }) {
-  const isDisabled = disabled || !available;
+  const [busy, setBusy] = useState(false);
+  const isDisabled = disabled || !available || busy;
   const ariaLabel = label || `${installed ? 'Uninstall' : 'Install'} ${itemId}`;
+
+  // Client mode — no form, no navigation.
+  if (typeof onToggle === 'function') {
+    return (
+      <button
+        className={`mkt-toggle ${installed ? 'on' : ''}`}
+        type="button"
+        role="switch"
+        aria-checked={installed}
+        aria-label={ariaLabel}
+        aria-busy={busy || undefined}
+        disabled={isDisabled}
+        onClick={async () => {
+          if (isDisabled) return;
+          setBusy(true);
+          try {
+            await onToggle(!installed);
+          } finally {
+            setBusy(false);
+          }
+        }}
+      />
+    );
+  }
+
+  // Server-action fallback.
   return (
     <form action={action} className="mkt-toggle-form">
       <input type="hidden" name="item_id" value={itemId} />
@@ -46,7 +81,7 @@ export default function InstallToggle({
         role="switch"
         aria-checked={installed}
         aria-label={ariaLabel}
-        disabled={isDisabled}
+        disabled={disabled || !available}
       />
     </form>
   );