@ateam-ai-mobile/ui 0.1.0 → 0.1.1

package/src/voice/VoiceCallSheet.tsx

@@ -0,0 +1,330 @@
+/**
+ * VoiceCallSheet — the full-screen voice call experience.
+ *
+ * AdaOrb hero + live transcript (with important-data highlighting) + mute
+ * control + an offscreen WebView that runs the actual WebRTC session (see
+ * useVoiceCall). Requires `voiceUrl` in the initAdas() config.
+ */
+import { useEffect, useRef, useMemo } from 'react';
+import {
+  View,
+  Text,
+  StyleSheet,
+  Platform,
+  Pressable,
+  ScrollView,
+} from 'react-native';
+import Svg, { Path, Line } from 'react-native-svg';
+import { WebView } from 'react-native-webview';
+import { ada } from '@ateam-ai-mobile/core';
+import { AdaOrb, SurfaceFullscreen, type AdaOrbState } from '@ateam-ai-mobile/plugin-runtime';
+import useVoiceCall from './useVoiceCall';
+
+interface Props {
+  visible: boolean;
+  onClose: () => void;
+}
+
+const fontFam = Platform.OS === 'ios' ? 'SF Pro Rounded' : undefined;
+
+// Detect important data in voice transcripts so the user can verify it visually.
+const HIGHLIGHT_PATTERNS = [
+  /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g,
+  /(?:\+?\d[\s\-().]?){7,}\d/g,
+  /\$\d[\d,]*(?:\.\d{1,2})?/g,
+  /\b\d{1,2}:\d{2}\s?(?:am|pm|AM|PM)?\b/g,
+  /\bhttps?:\/\/\S+/g,
+  /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/g,
+];
+
+function highlightSegments(text: string): Array<{ text: string; important: boolean }> {
+  const ranges: Array<[number, number]> = [];
+  for (const p of HIGHLIGHT_PATTERNS) {
+    p.lastIndex = 0;
+    let m;
+    while ((m = p.exec(text)) !== null) {
+      ranges.push([m.index, m.index + m[0].length]);
+    }
+  }
+  if (!ranges.length) return [{ text, important: false }];
+  ranges.sort((a, b) => a[0] - b[0]);
+  const merged: Array<[number, number]> = [];
+  for (const r of ranges) {
+    if (merged.length && r[0] <= merged[merged.length - 1][1]) {
+      merged[merged.length - 1][1] = Math.max(merged[merged.length - 1][1], r[1]);
+    } else {
+      merged.push([r[0], r[1]]);
+    }
+  }
+  const segments: Array<{ text: string; important: boolean }> = [];
+  let cursor = 0;
+  for (const [start, end] of merged) {
+    if (start > cursor) segments.push({ text: text.slice(cursor, start), important: false });
+    segments.push({ text: text.slice(start, end), important: true });
+    cursor = end;
+  }
+  if (cursor < text.length) segments.push({ text: text.slice(cursor), important: false });
+  return segments;
+}
+
+function HighlightedText({
+  text,
+  baseStyle,
+  importantStyle,
+}: {
+  text: string;
+  baseStyle: any;
+  importantStyle: any;
+}) {
+  const segments = useMemo(() => highlightSegments(text), [text]);
+  return (
+    <Text style={baseStyle}>
+      {segments.map((seg, i) => (
+        <Text key={i} style={seg.important ? importantStyle : undefined}>
+          {seg.text}
+        </Text>
+      ))}
+    </Text>
+  );
+}
+
+export default function VoiceCallSheet({ visible, onClose }: Props) {
+  const {
+    callState, aiPartial, userPartial, transcript, error, muted,
+    startCall, stopCall, toggleMute, handleWebViewMessage, getWebViewSource, webViewRef,
+    injectedTrackHook,
+  } = useVoiceCall();
+
+  const scrollRef = useRef<ScrollView>(null);
+
+  useEffect(() => {
+    if (visible && callState === 'idle') startCall();
+  }, [visible]); // eslint-disable-line react-hooks/exhaustive-deps
+
+  useEffect(() => {
+    scrollRef.current?.scrollToEnd?.({ animated: true });
+  }, [transcript.length, aiPartial, userPartial]);
+
+  const handleHangUp = () => {
+    stopCall();
+    onClose();
+  };
+
+  const orbState: AdaOrbState = useMemo(() => {
+    if (callState === 'connecting') return 'thinking';
+    if (callState === 'error') return 'idle';
+    if (aiPartial) return 'speaking';
+    if (userPartial) return 'listening';
+    return 'idle';
+  }, [callState, aiPartial, userPartial]);
+
+  const statusLabel = useMemo(() => {
+    if (callState === 'connecting') return 'Connecting…';
+    if (callState === 'error') return error || 'Couldn’t connect';
+    if (aiPartial) return 'Ada is speaking';
+    if (userPartial) return 'Listening';
+    if (callState === 'connected') return 'Go ahead, I’m here.';
+    return '';
+  }, [callState, aiPartial, userPartial, error]);
+
+  if (!visible) return null;
+
+  const visibleEntries = transcript.filter(e => e.role === 'user' || e.role === 'ai');
+
+  return (
+    <SurfaceFullscreen
+      visible={visible}
+      onClose={handleHangUp}
+      status={statusLabel}
+      hero={<AdaOrb size={180} state={orbState} />}
+      closeIcon="hangup"
+      closeLabel="End"
+    >
+      <ScrollView
+        ref={scrollRef}
+        style={styles.transcript}
+        contentContainerStyle={styles.transcriptContent}
+        showsVerticalScrollIndicator={false}
+      >
+        {callState === 'error' && error ? (
+          <View style={styles.errorBox}>
+            <Text style={styles.errorText}>{error}</Text>
+          </View>
+        ) : null}
+        {visibleEntries.map((entry, i) => (
+          <View key={`${entry.ts}-${i}`} style={styles.entry}>
+            <Text style={styles.role}>
+              {entry.role === 'user' ? 'YOU' : 'ADA'}
+            </Text>
+            <HighlightedText
+              text={entry.text}
+              baseStyle={styles.entryText}
+              importantStyle={styles.important}
+            />
+          </View>
+        ))}
+        {(userPartial || aiPartial) ? (
+          <View style={styles.entry}>
+            <Text style={styles.role}>
+              {userPartial ? 'YOU' : 'ADA'}
+            </Text>
+            <HighlightedText
+              text={userPartial || aiPartial}
+              baseStyle={styles.entryTextLive}
+              importantStyle={styles.important}
+            />
+          </View>
+        ) : null}
+      </ScrollView>
+
+      {/* Mute control — floating above transcript, only when call is live */}
+      {callState === 'connected' || callState === 'connecting' ? (
+        <View style={styles.muteWrap} pointerEvents="box-none">
+          <Pressable
+            onPress={toggleMute}
+            accessibilityRole="button"
+            accessibilityLabel={muted ? 'Unmute microphone' : 'Mute microphone'}
+            accessibilityState={{ selected: muted }}
+            style={[styles.muteBtn, muted && styles.muteBtnActive]}
+            hitSlop={12}
+          >
+            <Svg width={22} height={22} viewBox="0 0 24 24" fill="none" stroke={muted ? ada.paper : ada.ink} strokeWidth={2} strokeLinecap="round" strokeLinejoin="round">
+              <Path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+              <Path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+              <Line x1="12" y1="19" x2="12" y2="23" />
+              <Line x1="8" y1="23" x2="16" y2="23" />
+              {muted ? <Line x1="3" y1="3" x2="21" y2="21" stroke={ada.paper} strokeWidth={2.4} /> : null}
+            </Svg>
+          </Pressable>
+          <Text style={styles.muteLabel}>{muted ? 'Muted' : 'Mute'}</Text>
+        </View>
+      ) : null}
+
+      {/* WebView mounted offscreen — runs WebRTC silently */}
+      {callState !== 'idle' && (
+        <View style={styles.hiddenWebView} pointerEvents="none">
+          <WebView
+            ref={webViewRef}
+            source={getWebViewSource()}
+            style={{ flex: 1, backgroundColor: 'transparent' }}
+            onMessage={handleWebViewMessage}
+            onError={(e: any) => console.log('[VoiceWebView] error:', e.nativeEvent?.description)}
+            injectedJavaScriptBeforeContentLoaded={injectedTrackHook}
+            mediaPlaybackRequiresUserAction={false}
+            allowsInlineMediaPlayback={true}
+            mediaCapturePermissionGrantType="grant"
+            javaScriptEnabled={true}
+            originWhitelist={['*']}
+          />
+        </View>
+      )}
+    </SurfaceFullscreen>
+  );
+}
+
+const styles = StyleSheet.create({
+  transcript: {
+    flex: 1,
+    paddingHorizontal: 28,
+  },
+  transcriptContent: {
+    paddingTop: 8,
+    paddingBottom: 16,
+    gap: 14,
+  },
+  entry: {
+    flexDirection: 'row',
+    gap: 10,
+    alignItems: 'flex-start',
+  },
+  role: {
+    fontSize: 10,
+    fontFamily: fontFam,
+    fontWeight: '700',
+    color: ada.inkMuted,
+    letterSpacing: 1.4,
+    minWidth: 32,
+    paddingTop: 3,
+    opacity: 0.55,
+  },
+  entryText: {
+    flex: 1,
+    fontSize: 14,
+    fontFamily: fontFam,
+    fontWeight: '400',
+    color: ada.inkSoft,
+    lineHeight: 20,
+    letterSpacing: -0.1,
+    opacity: 0.42,
+  },
+  entryTextLive: {
+    flex: 1,
+    fontSize: 14,
+    fontFamily: fontFam,
+    fontWeight: '400',
+    color: ada.inkSoft,
+    lineHeight: 20,
+    letterSpacing: -0.1,
+    opacity: 0.7,
+  },
+  important: {
+    fontSize: 17,
+    fontFamily: fontFam,
+    fontWeight: '700',
+    color: ada.ink,
+    letterSpacing: -0.1,
+    opacity: 1,
+  },
+  errorBox: {
+    backgroundColor: 'rgba(239,68,68,0.12)',
+    borderColor: 'rgba(239,68,68,0.45)',
+    borderWidth: 1,
+    borderRadius: 10,
+    paddingVertical: 10,
+    paddingHorizontal: 14,
+    marginBottom: 12,
+  },
+  errorText: {
+    fontSize: 13,
+    fontFamily: fontFam,
+    fontWeight: '500',
+    color: '#fca5a5',
+    lineHeight: 18,
+  },
+  muteWrap: {
+    position: 'absolute',
+    bottom: Platform.OS === 'ios' ? 56 : 40,
+    left: 28,
+    alignItems: 'center',
+    gap: 6,
+  },
+  muteBtn: {
+    width: 56,
+    height: 56,
+    borderRadius: 28,
+    backgroundColor: ada.paperSoft,
+    borderWidth: 1,
+    borderColor: ada.border,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  muteBtnActive: {
+    backgroundColor: '#C15545',
+    borderColor: '#C15545',
+  },
+  muteLabel: {
+    fontSize: 12,
+    fontFamily: fontFam,
+    fontWeight: '500',
+    color: ada.inkMuted,
+    letterSpacing: 0.3,
+  },
+  hiddenWebView: {
+    position: 'absolute',
+    width: 1,
+    height: 1,
+    top: -100,
+    left: -100,
+    opacity: 0,
+  },
+});

package/src/voice/useVoiceCall.ts

@@ -0,0 +1,200 @@
+/**
+ * useVoiceCall — WebView-based voice hook.
+ *
+ * The WebView runs WebRTC + OpenAI Realtime internally (iOS WebView supports
+ * WebRTC); communication is postMessage (transcript/state events → RN).
+ * Requires `voiceUrl` in the initAdas() config — getWebViewSource() throws
+ * without it. Identity (token/tenant/actor) comes from the host-owned config.
+ */
+import { useState, useRef, useCallback } from 'react';
+import { getVoiceApiUrl, getToken, getTenant, getUser } from '@ateam-ai-mobile/core';
+
+export interface TranscriptEntry {
+  role: 'user' | 'ai' | 'tool';
+  text: string;
+  ts: number;
+}
+
+export type CallState = 'idle' | 'connecting' | 'connected' | 'error';
+
+// Installed BEFORE the voice app loads. Captures every audio track that
+// could possibly be sent to the peer connection so the mute button can flip
+// enabled=false on all of them, regardless of how the voice app obtained or
+// added the track.
+const INJECT_TRACK_HOOK = `
+(function() {
+  if (window._adaTrackHookInstalled) return;
+  window._adaTrackHookInstalled = true;
+  window._allAudioTracks = [];
+  window._adaPeerConnections = [];
+
+  function track(t) {
+    if (t && t.kind === 'audio' && window._allAudioTracks.indexOf(t) === -1) {
+      window._allAudioTracks.push(t);
+    }
+  }
+
+  // 1) Hook getUserMedia — captures mic streams at request time.
+  try {
+    if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+      const origGUM = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
+      navigator.mediaDevices.getUserMedia = async function(constraints) {
+        const stream = await origGUM(constraints);
+        if (constraints && constraints.audio) {
+          try { stream.getAudioTracks().forEach(track); } catch (e) {}
+        }
+        return stream;
+      };
+    }
+  } catch (e) {}
+
+  // 2) Hook RTCPeerConnection.addTrack — catches whatever the app sends.
+  try {
+    if (window.RTCPeerConnection) {
+      const origPC = window.RTCPeerConnection;
+      const origAddTrack = origPC.prototype.addTrack;
+      const origAddStream = origPC.prototype.addStream;
+      const origAddTransceiver = origPC.prototype.addTransceiver;
+
+      origPC.prototype.addTrack = function(t) {
+        if (t && t.kind === 'audio') track(t);
+        return origAddTrack.apply(this, arguments);
+      };
+      if (origAddStream) {
+        origPC.prototype.addStream = function(stream) {
+          try { stream.getAudioTracks().forEach(track); } catch (e) {}
+          return origAddStream.apply(this, arguments);
+        };
+      }
+      if (origAddTransceiver) {
+        origPC.prototype.addTransceiver = function(arg) {
+          if (arg && arg.kind === 'audio') {
+            // Transceiver created with kind 'audio' — sender track set later.
+          } else if (arg && arg.kind === undefined && arg.track && arg.track.kind === 'audio') {
+            track(arg.track);
+          }
+          const tx = origAddTransceiver.apply(this, arguments);
+          try {
+            if (tx && tx.sender && tx.sender.track && tx.sender.track.kind === 'audio') track(tx.sender.track);
+          } catch (e) {}
+          return tx;
+        };
+      }
+    }
+  } catch (e) {}
+
+  // 3) Helper exposed for direct calls.
+  window._adaSetMuted = function(muted) {
+    let n = 0;
+    if (Array.isArray(window._allAudioTracks)) {
+      window._allAudioTracks.forEach(function(t) {
+        try { t.enabled = !muted; n++; } catch (e) {}
+      });
+    }
+    return n;
+  };
+})(); true;
+`;
+
+const buildMuteScript = (muted: boolean) => `
+(function(muted) {
+  if (typeof window.setMuted === 'function') { window.setMuted(muted); }
+  let n = 0;
+  if (typeof window._adaSetMuted === 'function') {
+    n = window._adaSetMuted(muted);
+  }
+  // Last-resort scan if hook somehow missed.
+  if (n === 0 && Array.isArray(window._allAudioTracks)) {
+    window._allAudioTracks.forEach(function(t) {
+      try { t.enabled = !muted; n++; } catch (e) {}
+    });
+  }
+  console.log('[ada-mute]', muted ? 'MUTED' : 'UNMUTED', 'tracks=', n);
+})(${muted}); true;
+`;
+
+export default function useVoiceCall() {
+  const [callState, setCallState] = useState<CallState>('idle');
+  const [sessionId, setSessionId] = useState<string | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  const [transcript, setTranscript] = useState<TranscriptEntry[]>([]);
+  const [aiPartial, setAiPartial] = useState('');
+  const [userPartial, setUserPartial] = useState('');
+  const [muted, setMuted] = useState(false);
+  const webViewRef = useRef<any>(null);
+
+  /** Handle postMessage events from the WebView */
+  const handleWebViewMessage = useCallback((event: any) => {
+    try {
+      const data = JSON.parse(event.nativeEvent.data);
+      console.log('[VoiceWebView] message:', data.type, data.text?.slice?.(0, 50) || data.state || '');
+      switch (data.type) {
+        case 'callState':
+        case 'call-state':
+          setCallState(data.state);
+          if (data.sessionId) setSessionId(data.sessionId);
+          if (data.error) setError(data.error);
+          break;
+        case 'transcript':
+          setTranscript(prev => [...prev, { role: data.role, text: data.text, ts: Date.now() }]);
+          break;
+        case 'aiPartial':
+          setAiPartial(data.text);
+          break;
+        case 'userPartial':
+          setUserPartial(data.text);
+          break;
+        case 'clearPartials':
+          setAiPartial('');
+          setUserPartial('');
+          break;
+        case 'error':
+          setError(data.message);
+          setCallState('error');
+          break;
+      }
+    } catch {}
+  }, []);
+
+  const startCall = useCallback(() => {
+    setCallState('connecting');
+    setError(null);
+    setTranscript([]);
+    setAiPartial('');
+    setUserPartial('');
+  }, []);
+
+  const stopCall = useCallback(() => {
+    // Tell WebView to hang up
+    webViewRef.current?.injectJavaScript?.('window.hangUp && window.hangUp(); true;');
+    setCallState('idle');
+    setAiPartial('');
+    setUserPartial('');
+    setMuted(false);
+  }, []);
+
+  const toggleMute = useCallback(() => {
+    setMuted(prev => {
+      const next = !prev;
+      webViewRef.current?.injectJavaScript?.(buildMuteScript(next));
+      return next;
+    });
+  }, []);
+
+  /** Build the WebView source. Identity comes from the SDK config (host-owned
+   *  auth) — token, tenant, actorId are passed to the voice mini-app. */
+  const getWebViewSource = useCallback(() => {
+    const token = getToken();
+    const tenant = getTenant();
+    const actorId = getUser()?.actorId || '';
+    return {
+      uri: `${getVoiceApiUrl()}/app?mode=mini&embedded=rn&authToken=${encodeURIComponent(token || '')}&tenant=${encodeURIComponent(tenant)}&actorId=${encodeURIComponent(actorId)}`,
+    };
+  }, []);
+
+  return {
+    callState, sessionId, error, transcript, aiPartial, userPartial, muted,
+    startCall, stopCall, toggleMute, handleWebViewMessage, getWebViewSource, webViewRef,
+    injectedTrackHook: INJECT_TRACK_HOOK,
+  };
+}