@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0

package/src/components/VoiceImageInput.tsx

@@ -0,0 +1,178 @@
+import { uriToVoiceImageAttachment } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import React, { useCallback, useState } from 'react';
+import {
+  ActivityIndicator,
+  Pressable,
+  StyleSheet,
+  Text,
+  TextInput,
+  View,
+} from 'react-native';
+import { useAgentAssistant } from '../context/AgentAssistantContext';
+
+export type VoiceImageInputProps = {
+  /** Demo default image URL (HTTPS) for one-tap send. */
+  demoImageUrl?: string;
+};
+
+const IMAGE_ONLY_HINT =
+  'Please describe this image or tell me what you would like to know about it.';
+
+/**
+ * P0 test UI: send image (optional text) during an active voice session (SDK-2).
+ */
+export function VoiceImageInput({ demoImageUrl }: VoiceImageInputProps): React.ReactElement | null {
+  const {
+    theme,
+    interactionMode,
+    isVoiceChannelReady,
+    sendVoiceUserInput,
+  } = useAgentAssistant();
+  const [imageUrl, setImageUrl] = useState(demoImageUrl ?? '');
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [sentHint, setSentHint] = useState<string | null>(null);
+
+  const sendImage = useCallback(
+    async (url: string) => {
+      const trimmed = url.trim();
+      if (!trimmed) {
+        setError('Enter an image URL');
+        return;
+      }
+      if (!isVoiceChannelReady) {
+        setError('Voice channel is not ready yet — wait for Listening');
+        return;
+      }
+      setBusy(true);
+      setError(null);
+      setSentHint(null);
+      try {
+        const attachment = await uriToVoiceImageAttachment(trimmed, 'voice-image.jpg');
+        sendVoiceUserInput({
+          text: IMAGE_ONLY_HINT,
+          attachments: [attachment],
+          clientMessageId: `user_img_${Date.now()}`,
+        });
+        setSentHint('Image sent');
+      } catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        setError(message);
+        console.warn('[nxtlinq] sendVoiceUserInput image failed:', message);
+      } finally {
+        setBusy(false);
+      }
+    },
+    [isVoiceChannelReady, sendVoiceUserInput],
+  );
+
+  if (interactionMode !== 'voice') return null;
+
+  return (
+    <View
+      style={[
+        styles.box,
+        {
+          borderTopColor: theme.colors.border,
+          backgroundColor: theme.colors.background,
+          padding: theme.spacing.sm,
+        },
+      ]}
+    >
+      <Text style={{ color: theme.colors.mutedText, fontSize: theme.typography.captionSize }}>
+        Send image during voice (P0 test)
+      </Text>
+      {!isVoiceChannelReady ? (
+        <Text
+          style={{
+            color: theme.colors.mutedText,
+            fontSize: theme.typography.captionSize,
+            marginTop: theme.spacing.xs,
+          }}
+        >
+          Waiting for voice channel…
+        </Text>
+      ) : null}
+      <TextInput
+        value={imageUrl}
+        onChangeText={setImageUrl}
+        placeholder="HTTPS image URL or data URI"
+        placeholderTextColor={theme.colors.mutedText}
+        autoCapitalize="none"
+        style={[
+          styles.input,
+          {
+            borderColor: theme.colors.border,
+            color: theme.colors.assistantText,
+            marginTop: theme.spacing.xs,
+          },
+        ]}
+      />
+      {error ? (
+        <Text style={{ color: theme.colors.error, fontSize: theme.typography.captionSize }}>
+          {error}
+        </Text>
+      ) : null}
+      {sentHint ? (
+        <Text style={{ color: theme.colors.primary, fontSize: theme.typography.captionSize }}>
+          {sentHint}
+        </Text>
+      ) : null}
+      <View style={[styles.row, { marginTop: theme.spacing.xs }]}>
+        <Pressable
+          onPress={() => sendImage(imageUrl)}
+          disabled={busy || !isVoiceChannelReady}
+          style={({ pressed }) => [
+            styles.btn,
+            {
+              backgroundColor: theme.colors.primary,
+              opacity: pressed || busy || !isVoiceChannelReady ? 0.6 : 1,
+            },
+          ]}
+        >
+          {busy ? (
+            <ActivityIndicator color={theme.colors.primaryText} size="small" />
+          ) : (
+            <Text style={{ color: theme.colors.primaryText, fontWeight: '600' }}>Send image</Text>
+          )}
+        </Pressable>
+        {demoImageUrl ? (
+          <Pressable
+            onPress={() => sendImage(demoImageUrl)}
+            disabled={busy || !isVoiceChannelReady}
+            style={({ pressed }) => [
+              styles.btn,
+              {
+                backgroundColor: theme.colors.surface,
+                borderWidth: 1,
+                borderColor: theme.colors.border,
+                opacity: pressed || busy || !isVoiceChannelReady ? 0.6 : 1,
+              },
+            ]}
+          >
+            <Text style={{ color: theme.colors.assistantText }}>Product image</Text>
+          </Pressable>
+        ) : null}
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  box: { borderTopWidth: StyleSheet.hairlineWidth },
+  input: {
+    borderWidth: 1,
+    borderRadius: 8,
+    paddingHorizontal: 10,
+    paddingVertical: 8,
+    fontSize: 14,
+  },
+  row: { flexDirection: 'row', gap: 8 },
+  btn: {
+    paddingHorizontal: 14,
+    paddingVertical: 8,
+    borderRadius: 8,
+    minWidth: 72,
+    alignItems: 'center',
+  },
+});

package/src/components/VoiceWaveform.tsx

@@ -0,0 +1,84 @@
+import React, { useEffect, useRef, useState } from 'react';
+import { StyleSheet, View } from 'react-native';
+import { useAgentAssistant } from '../context/AgentAssistantContext';
+import {
+  WAVEFORM_ACTIVE_STATUSES,
+  WAVEFORM_AUDIBLE_HOLD_MS,
+  WAVEFORM_AUDIBLE_THRESHOLD,
+} from '../voice/voiceWaveformConstants';
+
+const BAR_COUNT = 12;
+
+/**
+ * Polls `getOutputAudioLevel()` while the assistant is speaking (Berify-aligned).
+ * JeannieVoiceWaveform uses `active && audible` — not during user mic / idle.
+ */
+export function VoiceWaveform(): React.ReactElement | null {
+  const { theme, interactionMode, voiceStatus, getOutputAudioLevel } = useAgentAssistant();
+  const [levels, setLevels] = useState<number[]>(() => Array(BAR_COUNT).fill(0));
+  const lastAudibleAtRef = useRef(0);
+
+  useEffect(() => {
+    if (interactionMode !== 'voice') {
+      setLevels(Array(BAR_COUNT).fill(0));
+      return;
+    }
+
+    const id = setInterval(() => {
+      const now = Date.now();
+      const statusActive = WAVEFORM_ACTIVE_STATUSES.has(voiceStatus);
+      const rawLevel = getOutputAudioLevel();
+      if (rawLevel > WAVEFORM_AUDIBLE_THRESHOLD) {
+        lastAudibleAtRef.current = now;
+      }
+      const audible = now - lastAudibleAtRef.current < WAVEFORM_AUDIBLE_HOLD_MS;
+      const effectiveActive = statusActive && audible;
+      const scaled = effectiveActive ? rawLevel : 0;
+      setLevels((prev) => [...prev.slice(1), scaled]);
+    }, 100);
+    return () => clearInterval(id);
+  }, [interactionMode, voiceStatus, getOutputAudioLevel]);
+
+  if (interactionMode !== 'voice') return null;
+
+  return (
+    <View
+      style={[
+        styles.row,
+        {
+          paddingHorizontal: theme.spacing.md,
+          paddingBottom: theme.spacing.sm,
+          backgroundColor: theme.colors.surface,
+        },
+      ]}
+    >
+      {levels.map((level, i) => (
+        <View
+          key={`bar-${i}`}
+          style={[
+            styles.bar,
+            {
+              height: 8 + level * 28,
+              backgroundColor: theme.colors.voiceSpeaking,
+              opacity: 0.35 + level * 0.65,
+            },
+          ]}
+        />
+      ))}
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  row: {
+    flexDirection: 'row',
+    alignItems: 'flex-end',
+    justifyContent: 'center',
+    gap: 3,
+    minHeight: 40,
+  },
+  bar: {
+    width: 4,
+    borderRadius: 2,
+  },
+});

package/src/context/AgentAssistantContext.tsx

@@ -0,0 +1,369 @@
+import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import {
+  useNxtlinqAgent,
+  useNxtlinqVoice,
+  type UseNxtlinqAgentResult,
+  type UseNxtlinqVoiceResult,
+} from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import React, {
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useState,
+  type ReactNode,
+} from 'react';
+import { defaultAgentAssistantTheme } from '../theme/defaultTheme';
+import type { AgentAssistantTheme, NxtlinqAgentAssistantProps, PresetMessage } from '../types';
+import { useVoiceMicState } from '../voice/useVoiceMicState';
+import { waitForIOSVoiceCaptureRelease } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import { isTextTtsPlayerSupported, TextTtsPlayer, type TextTtsPlayerHandle } from '../voice/TextTtsPlayer';
+
+export type InteractionMode = 'text' | 'voice';
+
+export type AgentAssistantContextValue = UseNxtlinqAgentResult &
+  UseNxtlinqVoiceResult & {
+  theme: AgentAssistantTheme;
+  title: string;
+  placeholder: string;
+  presetMessages: PresetMessage[];
+  interactionMode: InteractionMode;
+  setInteractionMode: (mode: InteractionMode) => void;
+  inputText: string;
+  setInputText: (text: string) => void;
+  isVoiceAvailable: boolean;
+  isVoiceConnecting: boolean;
+  isMicMuted: boolean;
+  isMicHeldForAssistant: boolean;
+  toggleVoiceMicMute: () => void;
+  sendText: () => Promise<void>;
+  selectPreset: (preset: PresetMessage) => Promise<void>;
+  /** True when voice data channel is open and ready for user_input. */
+  isVoiceChannelReady: boolean;
+  postTextTts: (text: string) => Promise<{ audio: ArrayBuffer; mimeType: string }>;
+  buildTextTtsPlaybackUri: (result: { audio: ArrayBuffer; mimeType: string }) => string;
+  playMessageTts: (messageId: string, text: string) => Promise<void>;
+  playingMessageId: string | null;
+  isTextTtsAvailable: boolean;
+};
+
+const AgentAssistantContext = createContext<AgentAssistantContextValue | null>(null);
+
+export type AgentAssistantProviderProps = {
+  children: ReactNode;
+  ui: Pick<
+    NxtlinqAgentAssistantProps,
+    | 'title'
+    | 'placeholder'
+    | 'presetMessages'
+    | 'enableVoice'
+    | 'theme'
+    | 'startWithMicMuted'
+    | 'holdMicDuringAssistant'
+    | 'textTtsVolume'
+    | 'voiceRemoteAudioGain'
+  > & {
+    webrtcEnabled: boolean;
+  };
+};
+
+export function AgentAssistantProvider({
+  children,
+  ui,
+}: AgentAssistantProviderProps): React.ReactElement {
+  const agent = useNxtlinqAgent();
+  const voice = useNxtlinqVoice();
+
+  const theme = useMemo(
+    () => ({
+      ...defaultAgentAssistantTheme,
+      ...ui.theme,
+      colors: { ...defaultAgentAssistantTheme.colors, ...ui.theme?.colors },
+      spacing: { ...defaultAgentAssistantTheme.spacing, ...ui.theme?.spacing },
+      radius: { ...defaultAgentAssistantTheme.radius, ...ui.theme?.radius },
+      typography: {
+        ...defaultAgentAssistantTheme.typography,
+        ...ui.theme?.typography,
+      },
+    }),
+    [ui.theme],
+  );
+
+  const [interactionMode, setInteractionModeState] = useState<InteractionMode>('text');
+  const [inputText, setInputText] = useState('');
+  const [isVoiceConnecting, setIsVoiceConnecting] = useState(false);
+  const [playingMessageId, setPlayingMessageId] = useState<string | null>(null);
+  const textTtsPlayerRef = React.useRef<TextTtsPlayerHandle | null>(null);
+  const ttsRequestIdRef = React.useRef(0);
+  const voiceConnectChainRef = React.useRef<Promise<unknown>>(Promise.resolve());
+
+  const isVoiceAvailable = Boolean(ui.webrtcEnabled && ui.enableVoice !== false);
+  const micStartsMuted = ui.startWithMicMuted ?? true;
+
+  const {
+    isMicMuted,
+    isMicHeldForAssistant,
+    toggleVoiceMicMute,
+    prepareForVoiceConnect,
+    resetMicState,
+    clearAssistantMicHold,
+  } = useVoiceMicState(voice, isVoiceConnecting, {
+    startWithMicMuted: micStartsMuted,
+    // Decoupled: open-mic demo can start unmuted yet still mute during assistant TTS.
+    holdMicDuringAssistant: ui.holdMicDuringAssistant ?? true,
+  });
+
+  const setInteractionMode = useCallback(
+    (mode: InteractionMode) => {
+      if (mode === 'text' && interactionMode === 'voice') {
+        void (async () => {
+          await voice.stopVoice('switch_to_text');
+          await waitForIOSVoiceCaptureRelease();
+          resetMicState();
+        })();
+      }
+      setInteractionModeState(mode);
+    },
+    [interactionMode, voice, resetMicState],
+  );
+
+  useEffect(() => {
+    if (interactionMode === 'text' && voice.voiceSessionId != null) {
+      void (async () => {
+        await voice.stopVoice('mode_text_cleanup');
+        await waitForIOSVoiceCaptureRelease();
+        resetMicState();
+      })();
+    }
+  }, [interactionMode, voice.voiceSessionId, voice, resetMicState]);
+
+  const sendText = useCallback(async () => {
+    const text = inputText.trim();
+    if (!text || agent.isLoading) return;
+    setInputText('');
+    await agent.sendMessage(text);
+  }, [agent, inputText]);
+
+  const selectPreset = useCallback(
+    async (preset: PresetMessage) => {
+      if (preset.autoSend) {
+        await agent.sendMessage(preset.text);
+      } else {
+        setInputText(preset.text);
+      }
+    },
+    [agent],
+  );
+
+  const wrappedStartVoice = useCallback(
+    (options?: Parameters<typeof voice.startVoice>[0]) => {
+      const connect = async () => {
+        if (voice.voiceSessionId != null) {
+          await voice.stopVoice('restart_voice');
+          await waitForIOSVoiceCaptureRelease();
+        }
+        prepareForVoiceConnect();
+        setInteractionModeState('voice');
+        setIsVoiceConnecting(true);
+        try {
+          const session = await voice.startVoice({
+            startWithMicMuted: micStartsMuted,
+            ...options,
+            onOpen: () => {
+              if (!micStartsMuted) {
+                voice.muteMic(false);
+              }
+              options?.onOpen?.();
+            },
+            onClose: (reason) => {
+              resetMicState();
+              const userInitiated =
+                reason === 'switch_to_text' ||
+                reason === 'client_stop' ||
+                reason === 'mode_text_cleanup';
+              if (userInitiated) {
+                setInteractionModeState('text');
+              } else {
+                console.warn('[nxtlinq] voice session closed:', reason);
+              }
+              options?.onClose?.(reason);
+            },
+            onError: (err) => {
+              resetMicState();
+              console.warn('[nxtlinq] voice session error:', err.message);
+              options?.onError?.(err);
+            },
+          });
+          return session;
+        } catch (err) {
+          setInteractionModeState('text');
+          resetMicState();
+          throw err;
+        } finally {
+          setIsVoiceConnecting(false);
+        }
+      };
+      const next = voiceConnectChainRef.current.then(connect, connect);
+      voiceConnectChainRef.current = next.catch(() => undefined);
+      return next;
+    },
+    [voice, prepareForVoiceConnect, resetMicState, micStartsMuted, ui.voiceRemoteAudioGain],
+  );
+
+  const wrappedStopVoice = useCallback(async () => {
+    await voice.stopVoice('client_stop');
+    await waitForIOSVoiceCaptureRelease();
+    resetMicState();
+    setInteractionModeState('text');
+  }, [voice, resetMicState]);
+
+  const wrappedInterrupt = useCallback(() => {
+    voice.interrupt();
+    clearAssistantMicHold();
+  }, [voice, clearAssistantMicHold]);
+
+  const isVoiceChannelReady =
+    interactionMode === 'voice' &&
+    voice.voiceSessionId != null &&
+    (Boolean(agent.agent.getVoiceSession()?.isAppChannelOpen()) ||
+      voice.voiceStatus === 'listening');
+
+  const postTextTts = useCallback(
+    (text: string) => agent.agent.postTextTts(text),
+    [agent.agent],
+  );
+
+  const buildTextTtsPlaybackUri = useCallback(
+    (result: { audio: ArrayBuffer; mimeType: string }) =>
+      agent.agent.buildTextTtsPlaybackUri(result),
+    [agent.agent],
+  );
+
+  const playMessageTts = useCallback(
+    async (messageId: string, text: string) => {
+      const trimmed = text.trim();
+      if (!trimmed) return;
+      const player = textTtsPlayerRef.current;
+      if (!player?.isAvailable) {
+        throw new Error(
+          'Text TTS playback requires react-native-video in your app (see Berify Jeannie).',
+        );
+      }
+      const requestId = ++ttsRequestIdRef.current;
+      player.stop();
+      setPlayingMessageId(messageId);
+      try {
+        const result = await postTextTts(trimmed);
+        if (ttsRequestIdRef.current !== requestId) return;
+        player.play(result);
+      } catch (err) {
+        if (ttsRequestIdRef.current === requestId) {
+          setPlayingMessageId(null);
+        }
+        throw err;
+      }
+    },
+    [postTextTts],
+  );
+
+  const isTextTtsAvailable = isTextTtsPlayerSupported();
+
+  const value = useMemo<AgentAssistantContextValue>(
+    () => ({
+      ...agent,
+      ...voice,
+      isVoiceActive:
+        interactionMode === 'voice' && voice.voiceSessionId != null,
+      startVoice: wrappedStartVoice,
+      stopVoice: wrappedStopVoice,
+      interrupt: wrappedInterrupt,
+      isMicMuted,
+      isMicHeldForAssistant,
+      toggleVoiceMicMute,
+      theme,
+      title: ui.title ?? 'AI Assistant',
+      placeholder: ui.placeholder ?? 'Type a message…',
+      presetMessages: ui.presetMessages ?? [],
+      interactionMode,
+      setInteractionMode,
+      inputText,
+      setInputText,
+      isVoiceAvailable,
+      isVoiceConnecting,
+      isVoiceChannelReady,
+      postTextTts,
+      buildTextTtsPlaybackUri,
+      playMessageTts,
+      playingMessageId,
+      isTextTtsAvailable,
+      sendText,
+      selectPreset,
+    }),
+    [
+      agent,
+      voice,
+      wrappedStartVoice,
+      wrappedStopVoice,
+      wrappedInterrupt,
+      isMicMuted,
+      isMicHeldForAssistant,
+      toggleVoiceMicMute,
+      theme,
+      ui.title,
+      ui.placeholder,
+      ui.presetMessages,
+      interactionMode,
+      inputText,
+      isVoiceAvailable,
+      isVoiceConnecting,
+      isVoiceChannelReady,
+      postTextTts,
+      buildTextTtsPlaybackUri,
+      playMessageTts,
+      playingMessageId,
+      isTextTtsAvailable,
+      sendText,
+      selectPreset,
+      voice.voiceStatus,
+    ],
+  );
+
+  return (
+    <AgentAssistantContext.Provider value={value}>
+      {children}
+      <TextTtsPlayer
+        playerRef={textTtsPlayerRef}
+        volume={ui.textTtsVolume ?? 1}
+        onPlayingChange={(playing) => {
+          if (!playing) setPlayingMessageId(null);
+        }}
+        onError={(message) => {
+          console.warn('[nxtlinq] TTS playback:', message);
+        }}
+      />
+    </AgentAssistantContext.Provider>
+  );
+}
+
+export function useAgentAssistant(): AgentAssistantContextValue {
+  const ctx = useContext(AgentAssistantContext);
+  if (!ctx) {
+    throw new Error(
+      'useAgentAssistant must be used within <NxtlinqAgentAssistant> or <AgentAssistantProvider>',
+    );
+  }
+  return ctx;
+}
+
+export function voiceStatusLabel(status: VoiceStatus): string {
+  const labels: Record<VoiceStatus, string> = {
+    idle: 'Ready',
+    listening: 'Listening',
+    transcribing: 'Transcribing',
+    thinking: 'Thinking',
+    generating: 'Generating',
+    speaking: 'Speaking',
+  };
+  return labels[status] ?? status;
+}

package/src/index.ts

@@ -0,0 +1,59 @@
+export { NxtlinqAgentAssistant } from './NxtlinqAgentAssistant';
+
+export type {
+  NxtlinqAgentAssistantProps,
+  AgentAssistantTheme,
+  PresetMessage,
+  Message,
+  ToolUse,
+} from './types';
+
+export { defaultAgentAssistantTheme } from './theme/defaultTheme';
+
+export {
+  AgentAssistantProvider,
+  useAgentAssistant,
+  voiceStatusLabel,
+  type AgentAssistantContextValue,
+  type InteractionMode,
+} from './context/AgentAssistantContext';
+
+export { AgentMessageList } from './components/AgentMessageList';
+export { AgentComposer } from './components/AgentComposer';
+export { AgentVoiceBar } from './components/AgentVoiceBar';
+export { AgentRemoteAudio } from './components/AgentRemoteAudio';
+export { AudioSessionWaker } from './voice/AudioSessionWaker';
+export { PresetMessageChips } from './components/PresetMessageChips';
+export { AgentAssistantShell } from './components/AgentAssistantShell';
+export { VoiceGreetTrigger } from './components/VoiceGreetTrigger';
+
+// Headless SDK re-exports for apps that compose custom layouts
+export {
+  NxtlinqAgentProvider,
+  useNxtlinqAgent,
+  useNxtlinqVoice,
+  createNxtlinqAgentRN,
+  type NxtlinqAgentProviderProps,
+  type RNWebRTCModule,
+} from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+
+export type {
+  AgentEnvironment,
+  AgentConfig,
+  Attachment,
+  AgentResponse,
+  SendMessageOptions,
+  NxtlinqAgentSnapshot,
+  StartVoiceSessionOptions,
+  VoiceSession,
+  VoiceStatus,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export {
+  NxtlinqAgent,
+  setApiHosts,
+  VoiceNotSupportedError,
+  mapServerHistoryToMessages,
+  appendServerHistoryIntoMessages,
+  STORAGE_KEYS,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';