@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0

package/src/react-native.d.ts

@@ -0,0 +1,42 @@
+declare module 'react-native' {
+  import type * as React from 'react';
+
+  export type StyleProp<T> = T | RecursiveArray<T | false | null | undefined> | false | null | undefined;
+  type RecursiveArray<T> = T | ReadonlyArray<RecursiveArray<T>>;
+  export type ViewStyle = Record<string, unknown>;
+
+  type PressableState = { pressed: boolean };
+  type PressableStyle = StyleProp<ViewStyle> | ((state: PressableState) => StyleProp<ViewStyle>);
+
+  export const View: React.ComponentType<{
+    style?: StyleProp<ViewStyle>;
+    children?: React.ReactNode;
+  }>;
+  export const Text: React.ComponentType<{
+    style?: StyleProp<ViewStyle>;
+    children?: React.ReactNode;
+    numberOfLines?: number;
+  }>;
+  export const TextInput: React.ComponentType<Record<string, unknown>>;
+  export const Pressable: React.ComponentType<{
+    onPress?: () => void;
+    disabled?: boolean;
+    style?: PressableStyle;
+    children?: React.ReactNode;
+  }>;
+  export const ScrollView: React.ComponentType<Record<string, unknown>>;
+  export const FlatList: <ItemT>(props: {
+    ref?: React.Ref<{ scrollToEnd: (opts: { animated: boolean }) => void }>;
+    data: ItemT[] | null | undefined;
+    keyExtractor?: (item: ItemT, index: number) => string;
+    renderItem: (info: { item: ItemT; index: number }) => React.ReactElement | null;
+    contentContainerStyle?: StyleProp<ViewStyle>;
+    ListEmptyComponent?: React.ReactElement | null;
+    ListFooterComponent?: React.ReactElement | null;
+  }) => React.ReactElement | null;
+  export const ActivityIndicator: React.ComponentType<Record<string, unknown>>;
+  export const StyleSheet: {
+    create<T extends Record<string, ViewStyle>>(styles: T): T;
+    hairlineWidth: number;
+  };
+}

package/src/theme/defaultTheme.ts

@@ -0,0 +1,35 @@
+import type { AgentAssistantTheme } from '../types';
+
+export const defaultAgentAssistantTheme: AgentAssistantTheme = {
+  colors: {
+    background: '#ffffff',
+    surface: '#f8fafc',
+    border: '#e2e8f0',
+    primary: '#2563eb',
+    primaryText: '#ffffff',
+    userBubble: '#2563eb',
+    userText: '#ffffff',
+    assistantBubble: '#f1f5f9',
+    assistantText: '#0f172a',
+    mutedText: '#64748b',
+    error: '#dc2626',
+    voiceActive: '#22c55e',
+    voiceSpeaking: '#ec4899',
+  },
+  spacing: {
+    xs: 4,
+    sm: 8,
+    md: 12,
+    lg: 16,
+  },
+  radius: {
+    bubble: 16,
+    panel: 12,
+    button: 8,
+  },
+  typography: {
+    titleSize: 17,
+    bodySize: 15,
+    captionSize: 12,
+  },
+};

package/src/types.ts

@@ -0,0 +1,107 @@
+import type { Message, ToolUse as CoreToolUse } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import type { NxtlinqAgentProviderProps } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import type { StyleProp, ViewStyle } from 'react-native';
+import type { VoiceAutoGreetConfig } from './voice/useVoiceAutoGreet';
+
+export type PresetMessage = {
+  text: string;
+  autoSend?: boolean;
+};
+
+export type AgentAssistantTheme = {
+  colors: {
+    background: string;
+    surface: string;
+    border: string;
+    primary: string;
+    primaryText: string;
+    userBubble: string;
+    userText: string;
+    assistantBubble: string;
+    assistantText: string;
+    mutedText: string;
+    error: string;
+    voiceActive: string;
+    voiceSpeaking: string;
+  };
+  spacing: {
+    xs: number;
+    sm: number;
+    md: number;
+    lg: number;
+  };
+  radius: {
+    bubble: number;
+    panel: number;
+    button: number;
+  };
+  typography: {
+    titleSize: number;
+    bodySize: number;
+    captionSize: number;
+  };
+};
+
+export type NxtlinqAgentAssistantProps = Omit<NxtlinqAgentProviderProps, 'children'> & {
+  children?: NxtlinqAgentProviderProps['children'];
+  /** Panel title in the header. */
+  title?: string;
+  placeholder?: string;
+  presetMessages?: PresetMessage[];
+  /** Fetch history on mount when `pseudoId` is set. */
+  loadHistoryOnMount?: boolean;
+  /** Max messages to prefetch (default 50). */
+  historyLast?: number;
+  /** Show voice mode controls (requires `webrtcModule`). */
+  enableVoice?: boolean;
+  /** Start in voice interaction mode. */
+  startInVoiceMode?: boolean;
+  /**
+   * Mute mic when voice connects (Berify hold-to-talk). Set false for open-mic demos.
+   * @default true
+   */
+  startWithMicMuted?: boolean;
+  /**
+   * Mute mic while assistant is transcribing/speaking (echo guard).
+   * Independent of `startWithMicMuted`; defaults true.
+   */
+  holdMicDuringAssistant?: boolean;
+  /** Show output RMS waveform while in voice mode (P0 / SDK-1). */
+  showVoiceWaveform?: boolean;
+  /** Show voice image URL input for P0 testing (SDK-2). */
+  showVoiceImageInput?: boolean;
+  /** Demo product image URL for greet + quick send. */
+  voiceDemoProductImageUrl?: string;
+  /** Auto-greet on first voice connect when no user history (SDK-3). */
+  voiceAutoGreet?: VoiceAutoGreetConfig | boolean;
+  /**
+   * iOS: bundled silent MP3 (`require('./assets/silent.mp3')`) to wake AVAudioSession.
+   * Strongly recommended — without it WebRTC/TTS may be silent on cold start.
+   */
+  iosSilentAudioSource?: number | { uri: string };
+  /**
+   * Voice-mode assistant playback gain (WebRTC `_setVolume`, 0–10). Default 5 to
+   * match text TTS loudness (react-native-video volume 1). Default 10.
+   */
+  voiceRemoteAudioGain?: number;
+  /** Text-mode TTS volume (react-native-video, 0–1). @default 1 */
+  textTtsVolume?: number;
+  theme?: Partial<AgentAssistantTheme>;
+  style?: StyleProp<ViewStyle>;
+  headerStyle?: StyleProp<ViewStyle>;
+  onMessage?: (message: Message) => void;
+  onError?: (error: Error) => void;
+  onToolUse?: (
+    toolUse: CoreToolUse,
+    onProgress?: (update: {
+      status?: string;
+      progress?: number;
+      partialResult?: string;
+      steps?: string[];
+      partialContent?: string;
+    }) => void,
+  ) => Promise<Message | void>;
+};
+
+export type ToolUse = CoreToolUse;
+export type { Message };

package/src/voice/AudioSessionWaker.tsx

@@ -0,0 +1,94 @@
+import {
+  markIOSAudioSessionReady,
+  setIOSAudioSessionPrewarmHandler,
+} from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import React, { useEffect, useRef } from 'react';
+
+type VideoSource = number | { uri: string };
+
+type VideoRef = {
+  seek: (time: number) => void;
+};
+
+type VideoModule = {
+  default: React.ComponentType<{
+    ref?: React.Ref<VideoRef>;
+    source: VideoSource;
+    paused?: boolean;
+    volume?: number;
+    playInBackground?: boolean;
+    repeat?: boolean;
+    ignoreSilentSwitch?: 'ignore' | 'obey' | 'inherit';
+    style?: object;
+    onLoad?: () => void;
+    onProgress?: (data: { currentTime: number }) => void;
+  }>;
+};
+
+function isIOS(): boolean {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { Platform } = require('react-native') as { Platform: { OS: string } };
+    return Platform.OS === 'ios';
+  } catch {
+    return false;
+  }
+}
+
+let VideoComponent: VideoModule['default'] | null = null;
+try {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  VideoComponent = (require('react-native-video') as VideoModule).default;
+} catch {
+  VideoComponent = null;
+}
+
+export type AudioSessionWakerProps = {
+  /**
+   * Bundled silent MP3 (e.g. `require('./assets/silent.mp3')`).
+   * Required on iOS for WebRTC + TTS until AVAudioSession is activated.
+   */
+  silentSource: VideoSource;
+};
+
+/**
+ * iOS-only: loop silent audio at volume 0 so react-native-video keeps AVAudioSession active.
+ * View stays mounted for app lifetime (Berify pattern — never unmount).
+ */
+export function AudioSessionWaker({ silentSource }: AudioSessionWakerProps): React.ReactElement | null {
+  const videoRef = useRef<VideoRef | null>(null);
+
+  useEffect(() => {
+    if (!isIOS() || !VideoComponent) return undefined;
+    // Seek-only replay — pausing react-native-video can drop AVAudioSession on repeated reconnects.
+    setIOSAudioSessionPrewarmHandler(() => {
+      try {
+        videoRef.current?.seek(0);
+      } catch {
+        /* noop */
+      }
+    });
+    return () => setIOSAudioSessionPrewarmHandler(null);
+  }, []);
+
+  if (!isIOS() || !VideoComponent) {
+    return null;
+  }
+
+  return (
+    <VideoComponent
+      ref={videoRef}
+      source={silentSource}
+      paused={false}
+      volume={0}
+      playInBackground={false}
+      repeat
+      ignoreSilentSwitch="ignore"
+      style={{ width: 0, height: 0, position: 'absolute', opacity: 0 }}
+      onLoad={() => markIOSAudioSessionReady()}
+      onProgress={({ currentTime }: { currentTime: number }) => {
+        if (currentTime > 0) markIOSAudioSessionReady();
+      }}
+    />
+  );
+}

package/src/voice/TextTtsPlayer.tsx

@@ -0,0 +1,151 @@
+import React, { useCallback, useRef, useState } from 'react';
+import { StyleSheet } from 'react-native';
+import type { PostTextTtsResult } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { isTtsCacheFileSupported, removeTtsCacheFile, writeTtsCacheFile } from './writeTtsCacheFile';
+
+type VideoModule = {
+  default: React.ComponentType<{
+    ref?: React.Ref<{ seek: (time: number) => void }>;
+    source?: { uri: string };
+    style?: object;
+    paused?: boolean;
+    volume?: number;
+    audioOnly?: boolean;
+    ignoreSilentSwitch?: 'ignore' | 'obey' | 'inherit';
+    playInBackground?: boolean;
+    playWhenInactive?: boolean;
+    onEnd?: () => void;
+    onError?: (e: { error?: { errorString?: string } }) => void;
+    bufferConfig?: Record<string, number>;
+    preferredForwardBufferDuration?: number;
+  }>;
+};
+
+let VideoComponent: VideoModule['default'] | null = null;
+try {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  VideoComponent = (require('react-native-video') as VideoModule).default;
+} catch {
+  VideoComponent = null;
+}
+
+export function isTextTtsPlayerSupported(): boolean {
+  return VideoComponent != null;
+}
+
+export type TextTtsPlayerHandle = {
+  play: (result: PostTextTtsResult) => Promise<void>;
+  stop: () => void;
+  isAvailable: boolean;
+};
+
+type Props = {
+  /** react-native-video volume 0–1. @default 1 */
+  volume?: number;
+  onPlayingChange?: (playing: boolean) => void;
+  onError?: (message: string) => void;
+  playerRef: React.MutableRefObject<TextTtsPlayerHandle | null>;
+};
+
+/**
+ * Hidden audio player for text-mode TTS (Berify uses react-native-video the same way).
+ */
+export function TextTtsPlayer({
+  volume = 1,
+  onPlayingChange,
+  onError,
+  playerRef,
+}: Props): React.ReactElement | null {
+  const videoRef = useRef<{ seek: (time: number) => void } | null>(null);
+  const [source, setSource] = useState<{ uri: string } | undefined>();
+  const lastFileRef = useRef<string | null>(null);
+
+  const stop = useCallback(() => {
+    const prev = lastFileRef.current;
+    lastFileRef.current = null;
+    setSource(undefined);
+    onPlayingChange?.(false);
+    if (prev && isTtsCacheFileSupported()) {
+      void removeTtsCacheFile(prev);
+    }
+  }, [onPlayingChange]);
+
+  const play = useCallback(
+    async (result: PostTextTtsResult) => {
+      if (!VideoComponent) return;
+      try {
+        const uri = await writeTtsCacheFile(result);
+        if (lastFileRef.current && lastFileRef.current !== uri) {
+          void removeTtsCacheFile(lastFileRef.current);
+        }
+        lastFileRef.current = uri;
+        setSource({ uri });
+        onPlayingChange?.(true);
+        requestAnimationFrame(() => {
+          try {
+            videoRef.current?.seek(0);
+          } catch {
+            /* noop */
+          }
+        });
+      } catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        onError?.(message);
+        stop();
+      }
+    },
+    [onPlayingChange, onError, stop],
+  );
+
+  React.useEffect(() => {
+    playerRef.current = {
+      play,
+      stop,
+      isAvailable: VideoComponent != null,
+    };
+    return () => {
+      playerRef.current = null;
+      stop();
+    };
+  }, [play, stop, playerRef]);
+
+  if (!VideoComponent) return null;
+
+  return (
+    <VideoComponent
+      ref={videoRef}
+      source={source}
+      style={styles.hidden}
+      paused={!source}
+      volume={volume}
+      audioOnly
+      ignoreSilentSwitch="ignore"
+      playInBackground={false}
+      playWhenInactive
+      onEnd={stop}
+      onError={(e) => {
+        const msg = e?.error?.errorString ?? 'TTS playback failed';
+        onError?.(msg);
+        stop();
+      }}
+      bufferConfig={{
+        minBufferMs: 800,
+        maxBufferMs: 4000,
+        bufferForPlaybackMs: 150,
+        bufferForPlaybackAfterRebufferMs: 500,
+      }}
+      preferredForwardBufferDuration={0.2}
+    />
+  );
+}
+
+const styles = StyleSheet.create({
+  hidden: {
+    width: 1,
+    height: 1,
+    opacity: 0.01,
+    position: 'absolute',
+    left: -1000,
+    top: -1000,
+  },
+});

package/src/voice/VoiceAutoGreetBinder.tsx

@@ -0,0 +1,38 @@
+import { uriToVoiceImageAttachment } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import React, { useCallback, useMemo } from 'react';
+import { useAgentAssistant } from '../context/AgentAssistantContext';
+import type { VoiceAutoGreetConfig } from './useVoiceAutoGreet';
+import { useVoiceAutoGreet } from './useVoiceAutoGreet';
+
+export function VoiceAutoGreetBinder({
+  config,
+  historyReady,
+}: {
+  config: VoiceAutoGreetConfig;
+  historyReady: boolean;
+}): null {
+  const voice = useAgentAssistant();
+  const hasUserMessage = useMemo(
+    () => voice.messages.some((m) => m.role === 'user'),
+    [voice.messages],
+  );
+
+  const resolveAttachments = useCallback(async () => {
+    const url = config.productImageUrl?.trim();
+    if (!url) return config.attachments;
+    const image = await uriToVoiceImageAttachment(url, 'product.jpg');
+    return config.attachments?.length
+      ? [...config.attachments, image]
+      : [image];
+  }, [config.attachments, config.productImageUrl]);
+
+  useVoiceAutoGreet(voice, {
+    enabled: true,
+    config,
+    historyReady,
+    hasUserMessage,
+    resolveAttachments: config.productImageUrl ? resolveAttachments : undefined,
+  });
+
+  return null;
+}

package/src/voice/useVoiceAutoGreet.ts

@@ -0,0 +1,95 @@
+import type { Attachment } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useEffect, useRef } from 'react';
+import type { UseNxtlinqVoiceResult } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+
+export type VoiceAutoGreetConfig = {
+  /** Greeting user text (e.g. product intro prompt). */
+  text?: string;
+  productName?: string;
+  /** HTTPS or data URI attached on first greet. */
+  productImageUrl?: string;
+  /** Pre-built attachments (overrides productImageUrl). */
+  attachments?: Attachment[];
+  skipUserMessage?: boolean;
+};
+
+function buildGreetingText(config: VoiceAutoGreetConfig): string {
+  if (config.text?.trim()) return config.text.trim();
+  const product = config.productName?.trim();
+  return product
+    ? `Hi! Please give me a brief introduction to ${product}.`
+    : 'Hi! Please say hello and introduce yourself.';
+}
+
+/**
+ * Berify Jeannie auto-greet: after voice connects + history loaded, trigger one
+ * synthetic user turn when there is no prior real user message.
+ */
+export function useVoiceAutoGreet(
+  voice: Pick<
+    UseNxtlinqVoiceResult,
+    'isVoiceActive' | 'triggerVoiceGreeting'
+  >,
+  options: {
+    enabled: boolean;
+    config?: VoiceAutoGreetConfig;
+    historyReady: boolean;
+    hasUserMessage: boolean;
+    resolveAttachments?: () => Promise<Attachment[] | undefined>;
+  },
+): void {
+  const sentRef = useRef(false);
+
+  useEffect(() => {
+    if (!options.enabled || !options.config) {
+      sentRef.current = false;
+      return;
+    }
+    if (!voice.isVoiceActive || !options.historyReady) return;
+    if (sentRef.current) return;
+    if (options.hasUserMessage) {
+      sentRef.current = true;
+      return;
+    }
+
+    let cancelled = false;
+    sentRef.current = true;
+
+    (async () => {
+      try {
+        const attachments =
+          options.config?.attachments ??
+          (await options.resolveAttachments?.());
+        if (cancelled) return;
+        await voice.triggerVoiceGreeting(
+          {
+            text: buildGreetingText(options.config!),
+            attachments,
+            skipUserMessage: options.config?.skipUserMessage ?? true,
+          },
+          { waitForChannel: true, timeoutMs: 12000 },
+        );
+      } catch {
+        sentRef.current = false;
+      }
+    })();
+
+    return () => {
+      cancelled = true;
+    };
+  }, [
+    voice.isVoiceActive,
+    voice.triggerVoiceGreeting,
+    options.enabled,
+    options.config,
+    options.historyReady,
+    options.hasUserMessage,
+    options.resolveAttachments,
+  ]);
+
+  useEffect(() => {
+    if (!voice.isVoiceActive) {
+      sentRef.current = false;
+    }
+  }, [voice.isVoiceActive]);
+}

package/src/voice/useVoiceMicState.ts

@@ -0,0 +1,116 @@
+import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import type { UseNxtlinqVoiceResult } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { ASSISTANT_MIC_HOLD_STATUSES } from './voiceMicConstants';
+
+/**
+ * Mic mute / hold behavior aligned with web SDK {@link useVoiceMode}.
+ * - Starts muted on connect (`startWithMicMuted`)
+ * - Holds mic during assistant transcribing → speaking
+ */
+export type UseVoiceMicStateOptions = {
+  /** When false, mic is open after connect (demo-friendly). Default true (Berify hold-to-talk). */
+  startWithMicMuted?: boolean;
+  /**
+   * When false, do not mute the mic (local + server) while the assistant is responding.
+   * Use with open-mic demos (`startWithMicMuted={false}`). Berify hold-to-talk keeps this true.
+   */
+  holdMicDuringAssistant?: boolean;
+};
+
+export function useVoiceMicState(
+  voice: UseNxtlinqVoiceResult,
+  isVoiceConnecting: boolean,
+  options?: UseVoiceMicStateOptions,
+) {
+  const connectMuted = options?.startWithMicMuted !== false;
+  const holdDuringAssistant = options?.holdMicDuringAssistant !== false;
+  const userMicMutedRef = useRef(connectMuted);
+  const assistantMicHoldRef = useRef(false);
+  const userMicOptInRef = useRef(!connectMuted);
+  const [isMicMuted, setIsMicMuted] = useState(connectMuted);
+
+  const applyMicState = useCallback(() => {
+    const shouldMute = userMicMutedRef.current || assistantMicHoldRef.current;
+    voice.muteMic(shouldMute);
+    setIsMicMuted(shouldMute);
+  }, [voice]);
+
+  const resetMicState = useCallback(() => {
+    userMicMutedRef.current = false;
+    assistantMicHoldRef.current = false;
+    userMicOptInRef.current = false;
+    setIsMicMuted(false);
+  }, []);
+
+  const prepareForVoiceConnect = useCallback(() => {
+    userMicMutedRef.current = connectMuted;
+    userMicOptInRef.current = !connectMuted;
+    assistantMicHoldRef.current = false;
+    setIsMicMuted(connectMuted);
+    voice.muteMic(connectMuted);
+  }, [voice, connectMuted]);
+
+  useEffect(() => {
+    if (!isVoiceConnecting) return;
+    userMicMutedRef.current = connectMuted;
+    voice.muteMic(connectMuted);
+    setIsMicMuted(connectMuted);
+  }, [isVoiceConnecting, voice, connectMuted]);
+
+  const prevVoiceStatusRef = useRef(voice.voiceStatus);
+
+  useEffect(() => {
+    const status = voice.voiceStatus;
+    const prev = prevVoiceStatusRef.current;
+    prevVoiceStatusRef.current = status;
+
+    if (holdDuringAssistant && ASSISTANT_MIC_HOLD_STATUSES.has(status)) {
+      assistantMicHoldRef.current = true;
+      applyMicState();
+      return;
+    }
+    if (status === 'listening' || status === 'idle') {
+      assistantMicHoldRef.current = false;
+      if (
+        connectMuted
+        && status === 'listening'
+        && prev === 'speaking'
+        && !userMicOptInRef.current
+      ) {
+        userMicMutedRef.current = true;
+      }
+      applyMicState();
+    }
+  }, [voice.voiceStatus, applyMicState, holdDuringAssistant]);
+
+  const toggleVoiceMicMute = useCallback(() => {
+    if (!voice.isVoiceActive && !isVoiceConnecting) return;
+    if (assistantMicHoldRef.current && userMicMutedRef.current) return;
+    if (assistantMicHoldRef.current) {
+      userMicMutedRef.current = true;
+      applyMicState();
+      return;
+    }
+    const nextMuted = !userMicMutedRef.current;
+    userMicMutedRef.current = nextMuted;
+    userMicOptInRef.current = !nextMuted;
+    applyMicState();
+  }, [voice.isVoiceActive, isVoiceConnecting, applyMicState]);
+
+  const clearAssistantMicHold = useCallback(() => {
+    assistantMicHoldRef.current = false;
+    applyMicState();
+  }, [applyMicState]);
+
+  const isMicHeldForAssistant = ASSISTANT_MIC_HOLD_STATUSES.has(voice.voiceStatus);
+
+  return {
+    isMicMuted,
+    isMicHeldForAssistant,
+    toggleVoiceMicMute,
+    prepareForVoiceConnect,
+    resetMicState,
+    clearAssistantMicHold,
+  };
+}

package/src/voice/voiceMicConstants.ts

@@ -0,0 +1,14 @@
+import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+/** Pause mic while assistant pipeline runs (aligned with web {@link useVoiceMode}). */
+export const ASSISTANT_MIC_HOLD_STATUSES: ReadonlySet<VoiceStatus> = new Set([
+  'transcribing',
+  'thinking',
+  'generating',
+  'speaking',
+]);
+
+export const SPEAKER_ACTIVE_STATUSES: ReadonlySet<VoiceStatus> = new Set([
+  'generating',
+  'speaking',
+]);