@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0 → 0.3.1

package/src/voice/useVoiceMicState.ts

@@ -16,6 +16,8 @@ export type UseVoiceMicStateOptions = {
    * Use with open-mic demos (`startWithMicMuted={false}`). Berify hold-to-talk keeps this true.
    */
   holdMicDuringAssistant?: boolean;
+  /** Called when user opens mic while assistant is responding (barge-in / interrupt). */
+  onBargeIn?: () => void;
 };
 
 export function useVoiceMicState(
@@ -29,18 +31,29 @@ export function useVoiceMicState(
   const assistantMicHoldRef = useRef(false);
   const userMicOptInRef = useRef(!connectMuted);
   const [isMicMuted, setIsMicMuted] = useState(connectMuted);
+  const [isCaptureActive, setIsCaptureActive] = useState(!connectMuted);
+
+  const syncCaptureActive = useCallback(() => {
+    const userOpen = !userMicMutedRef.current;
+    const holdBlocks =
+      assistantMicHoldRef.current
+      && !(userMicOptInRef.current && userOpen);
+    setIsCaptureActive(userOpen && !holdBlocks);
+  }, []);
 
   const applyMicState = useCallback(() => {
     const shouldMute = userMicMutedRef.current || assistantMicHoldRef.current;
     voice.muteMic(shouldMute);
     setIsMicMuted(shouldMute);
-  }, [voice]);
+    syncCaptureActive();
+  }, [voice, syncCaptureActive]);
 
   const resetMicState = useCallback(() => {
     userMicMutedRef.current = false;
     assistantMicHoldRef.current = false;
     userMicOptInRef.current = false;
     setIsMicMuted(false);
+    setIsCaptureActive(false);
   }, []);
 
   const prepareForVoiceConnect = useCallback(() => {
@@ -48,6 +61,7 @@ export function useVoiceMicState(
     userMicOptInRef.current = !connectMuted;
     assistantMicHoldRef.current = false;
     setIsMicMuted(connectMuted);
+    setIsCaptureActive(!connectMuted);
     voice.muteMic(connectMuted);
   }, [voice, connectMuted]);
 
@@ -56,6 +70,7 @@ export function useVoiceMicState(
     userMicMutedRef.current = connectMuted;
     voice.muteMic(connectMuted);
     setIsMicMuted(connectMuted);
+    setIsCaptureActive(!connectMuted);
   }, [isVoiceConnecting, voice, connectMuted]);
 
   const prevVoiceStatusRef = useRef(voice.voiceStatus);
@@ -66,6 +81,11 @@ export function useVoiceMicState(
     prevVoiceStatusRef.current = status;
 
     if (holdDuringAssistant && ASSISTANT_MIC_HOLD_STATUSES.has(status)) {
+      if (!userMicMutedRef.current && userMicOptInRef.current) {
+        assistantMicHoldRef.current = false;
+        applyMicState();
+        return;
+      }
       assistantMicHoldRef.current = true;
       applyMicState();
       return;
@@ -84,9 +104,22 @@ export function useVoiceMicState(
     }
   }, [voice.voiceStatus, applyMicState, holdDuringAssistant]);
 
+  const muteAfterSilenceCommit = useCallback(() => {
+    userMicMutedRef.current = true;
+    userMicOptInRef.current = false;
+    applyMicState();
+  }, [applyMicState]);
+
   const toggleVoiceMicMute = useCallback(() => {
     if (!voice.isVoiceActive && !isVoiceConnecting) return;
-    if (assistantMicHoldRef.current && userMicMutedRef.current) return;
+    if (assistantMicHoldRef.current && userMicMutedRef.current) {
+      assistantMicHoldRef.current = false;
+      userMicMutedRef.current = false;
+      userMicOptInRef.current = true;
+      options?.onBargeIn?.();
+      applyMicState();
+      return;
+    }
     if (assistantMicHoldRef.current) {
       userMicMutedRef.current = true;
       applyMicState();
@@ -96,7 +129,7 @@ export function useVoiceMicState(
     userMicMutedRef.current = nextMuted;
     userMicOptInRef.current = !nextMuted;
     applyMicState();
-  }, [voice.isVoiceActive, isVoiceConnecting, applyMicState]);
+  }, [voice.isVoiceActive, isVoiceConnecting, applyMicState, options]);
 
   const clearAssistantMicHold = useCallback(() => {
     assistantMicHoldRef.current = false;
@@ -107,8 +140,10 @@ export function useVoiceMicState(
 
   return {
     isMicMuted,
+    isCaptureActive,
     isMicHeldForAssistant,
     toggleVoiceMicMute,
+    muteAfterSilenceCommit,
     prepareForVoiceConnect,
     resetMicState,
     clearAssistantMicHold,

package/src/voice/useVoiceSilenceCommit.ts

@@ -0,0 +1,94 @@
+import type { VoiceSession, VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useCallback, useRef } from 'react';
+import { ASSISTANT_MIC_HOLD_STATUSES } from './voiceMicConstants';
+import {
+  MIC_BARGE_IN_RMS_THRESHOLD,
+  MIC_SILENCE_COMMIT_MS,
+  MIC_SILENCE_POLL_MS,
+  MIC_SPEECH_RMS_THRESHOLD,
+} from './voiceSilenceConstants';
+
+export function useVoiceSilenceCommit(
+  getSession: () => VoiceSession | null,
+  onMutedAfterCommit: () => void,
+  voiceStatus: VoiceStatus,
+) {
+  const lastSpeechAtRef = useRef(0);
+  const hadSpeechRef = useRef(false);
+  const commitInFlightRef = useRef(false);
+  const skipCommitOnMuteRef = useRef(false);
+  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const voiceStatusRef = useRef(voiceStatus);
+  voiceStatusRef.current = voiceStatus;
+  const onMutedAfterCommitRef = useRef(onMutedAfterCommit);
+  onMutedAfterCommitRef.current = onMutedAfterCommit;
+
+  const clearPoll = useCallback(() => {
+    if (pollRef.current) {
+      clearInterval(pollRef.current);
+      pollRef.current = null;
+    }
+  }, []);
+
+  const tryCommit = useCallback(
+    (reason: 'silence' | 'manual') => {
+      const session = getSession();
+      if (commitInFlightRef.current) return;
+      if (!hadSpeechRef.current) {
+        session?.clearInputAudio?.();
+        return;
+      }
+      commitInFlightRef.current = true;
+      hadSpeechRef.current = false;
+      session?.commitInputAudio?.();
+      if (reason === 'silence') {
+        skipCommitOnMuteRef.current = true;
+        onMutedAfterCommitRef.current();
+      }
+      queueMicrotask(() => {
+        commitInFlightRef.current = false;
+      });
+    },
+    [getSession],
+  );
+
+  const startPoll = useCallback(() => {
+    clearPoll();
+    pollRef.current = setInterval(() => {
+      if (commitInFlightRef.current || !hadSpeechRef.current) return;
+      if (Date.now() - lastSpeechAtRef.current < MIC_SILENCE_COMMIT_MS) return;
+      tryCommit('silence');
+    }, MIC_SILENCE_POLL_MS);
+  }, [clearPoll, tryCommit]);
+
+  const resetTurn = useCallback(() => {
+    hadSpeechRef.current = false;
+    lastSpeechAtRef.current = Date.now();
+    getSession()?.clearInputAudio?.();
+  }, [getSession]);
+
+  const onSpeechRms = useCallback((rms: number) => {
+    const threshold = ASSISTANT_MIC_HOLD_STATUSES.has(voiceStatusRef.current)
+      ? MIC_BARGE_IN_RMS_THRESHOLD
+      : MIC_SPEECH_RMS_THRESHOLD;
+    if (rms >= threshold) {
+      lastSpeechAtRef.current = Date.now();
+      hadSpeechRef.current = true;
+    }
+  }, []);
+
+  const consumeSkipCommitOnMute = useCallback(() => {
+    const skip = skipCommitOnMuteRef.current;
+    skipCommitOnMuteRef.current = false;
+    return skip;
+  }, []);
+
+  return {
+    startPoll,
+    clearPoll,
+    resetTurn,
+    onSpeechRms,
+    tryCommit,
+    consumeSkipCommitOnMute,
+  };
+}

package/src/voice/useVoiceTranscriptMessages.ts

@@ -0,0 +1,173 @@
+import type {
+  Message,
+  VoiceDoneEvent,
+  VoiceTranscriptEvent,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { mergeStreamingTranscript } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useCallback, useRef } from 'react';
+import type { InteractionMode } from '../context/AgentAssistantContext';
+
+type VoiceTranscriptAgentApi = {
+  getMessages: () => Message[];
+  setMessages: (messages: Message[]) => void;
+  syncVoiceTurnHistory: (options?: { last?: number }) => Promise<void>;
+};
+
+const STREAM_PREFIX = 'voice-stream-';
+
+function voiceMeta(sessionId: string | null) {
+  return {
+    voiceRealtime: true as const,
+    voiceSessionId: sessionId ?? undefined,
+  };
+}
+
+export function useVoiceTranscriptMessages(
+  api: VoiceTranscriptAgentApi,
+  interactionMode: InteractionMode,
+  voiceSessionId: string | null,
+) {
+  const streamIdRef = useRef<string | null>(null);
+  const sessionIdRef = useRef(voiceSessionId);
+  sessionIdRef.current = voiceSessionId;
+
+  const isVoiceUiActive = useCallback(
+    () => interactionMode === 'voice' && sessionIdRef.current != null,
+    [interactionMode],
+  );
+
+  const upsertStreaming = useCallback(
+    (text: string) => {
+      const messages = api.getMessages();
+      let streamId = streamIdRef.current;
+      if (!streamId) {
+        streamId = `${STREAM_PREFIX}${Date.now()}`;
+        streamIdRef.current = streamId;
+      }
+      const idx = messages.findIndex((m) => m.id === streamId);
+      const partialContent =
+        idx >= 0
+          ? mergeStreamingTranscript(messages[idx]?.partialContent ?? '', text)
+          : text;
+      const meta = voiceMeta(sessionIdRef.current);
+      if (idx >= 0) {
+        api.setMessages(
+          messages.map((m, i) =>
+            i === idx
+              ? { ...m, partialContent, isStreaming: true, metadata: { ...m.metadata, ...meta } }
+              : m,
+          ),
+        );
+        return;
+      }
+      api.setMessages([
+        ...messages,
+        {
+          id: streamId,
+          role: 'assistant',
+          content: '',
+          partialContent,
+          isStreaming: true,
+          timestamp: new Date().toISOString(),
+          metadata: meta,
+        },
+      ]);
+    },
+    [api],
+  );
+
+  const finalizeAssistant = useCallback(
+    (text: string, messageId?: string | null) => {
+      const trimmed = text.trim();
+      streamIdRef.current = null;
+      if (!trimmed) return;
+
+      const messages = api.getMessages();
+      const streamIdx = messages.findIndex((m) => m.isStreaming && m.role === 'assistant');
+      if (streamIdx >= 0) {
+        api.setMessages(
+          messages.map((m, i) =>
+            i === streamIdx
+              ? {
+                  ...m,
+                  id: messageId ?? m.id,
+                  content: trimmed,
+                  partialContent: undefined,
+                  isStreaming: false,
+                  metadata: { ...m.metadata, ...voiceMeta(sessionIdRef.current) },
+                }
+              : m,
+          ),
+        );
+        return;
+      }
+      const last = messages[messages.length - 1];
+      if (last?.role === 'assistant' && last.content === trimmed) return;
+      api.setMessages([
+        ...messages,
+        {
+          id: messageId ?? `voice-asst-${Date.now()}`,
+          role: 'assistant',
+          content: trimmed,
+          timestamp: new Date().toISOString(),
+          metadata: voiceMeta(sessionIdRef.current),
+        },
+      ]);
+    },
+    [api],
+  );
+
+  const handleTranscript = useCallback(
+    (event: VoiceTranscriptEvent) => {
+      if (!isVoiceUiActive()) return;
+      const text = event.text?.trim() ?? '';
+      if (event.role === 'assistant') {
+        // Keep one streaming bubble for the whole turn; finalize only in handleDone.
+        if (text) upsertStreaming(text);
+        return;
+      }
+      if (event.role === 'user' && !event.interim && text) {
+        const messages = api.getMessages();
+        const last = messages[messages.length - 1];
+        if (last?.role === 'user' && last.content === text) return;
+        api.setMessages([
+          ...messages,
+          {
+            id: `voice-user-${Date.now()}`,
+            role: 'user',
+            content: text,
+            timestamp: new Date().toISOString(),
+            metadata: voiceMeta(sessionIdRef.current),
+          },
+        ]);
+      }
+    },
+    [api, finalizeAssistant, isVoiceUiActive, upsertStreaming],
+  );
+
+  const handleDone = useCallback(
+    (event: VoiceDoneEvent) => {
+      if (!isVoiceUiActive()) return;
+      if (event.guardrailsBlocked || event.billingBlocked || event.error) {
+        streamIdRef.current = null;
+        return;
+      }
+      const reply = event.replyText?.trim() ?? '';
+      if (reply) {
+        finalizeAssistant(reply, event.assistantMessageId ?? undefined);
+      } else {
+        streamIdRef.current = null;
+      }
+      void api.syncVoiceTurnHistory({ last: 20 }).catch((err) => {
+        console.warn('[nxtlinq] syncVoiceTurnHistory after voice turn failed', err);
+      });
+    },
+    [api, finalizeAssistant, isVoiceUiActive],
+  );
+
+  const clearVoiceStream = useCallback(() => {
+    streamIdRef.current = null;
+  }, []);
+
+  return { handleTranscript, handleDone, clearVoiceStream };
+}

package/src/voice/useWsRealtimeAudio.ts

@@ -0,0 +1,200 @@
+import type { VoiceSession, VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import type { UseNxtlinqVoiceOptions } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import { waitForIOSAudioSessionReady } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
+import { useCallback, useEffect, useRef } from 'react';
+import { useVoiceSilenceCommit } from './useVoiceSilenceCommit';
+import { WsPcmPlayer } from './wsPcmPlayer';
+import { WsPcmRecorder } from './wsPcmRecorder';
+
+type WsVoiceCallbacks = Pick<
+  UseNxtlinqVoiceOptions,
+  'onOpen' | 'onAudioDelta' | 'onClose' | 'onError'
+>;
+
+export type UseWsRealtimeAudioOptions = {
+  voiceStatus: VoiceStatus;
+  muteAfterSilenceCommit: () => void;
+};
+
+function isIOS(): boolean {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { Platform } = require('react-native') as { Platform: { OS: string } };
+    return Platform.OS === 'ios';
+  } catch {
+    return false;
+  }
+}
+
+export function useWsRealtimeAudio(
+  isCaptureActive: boolean,
+  isVoiceActive: boolean,
+  options: UseWsRealtimeAudioOptions,
+) {
+  const playerRef = useRef<WsPcmPlayer | null>(null);
+  const recorderRef = useRef<WsPcmRecorder | null>(null);
+  const sessionRef = useRef<VoiceSession | null>(null);
+  const isCaptureActiveRef = useRef(isCaptureActive);
+  isCaptureActiveRef.current = isCaptureActive;
+  const prevCaptureActiveRef = useRef(isCaptureActive);
+  const iosSessionPrimedRef = useRef(false);
+
+  const getSession = useCallback(() => sessionRef.current, []);
+  const muteAfterSilenceCommitRef = useRef(options.muteAfterSilenceCommit);
+  muteAfterSilenceCommitRef.current = options.muteAfterSilenceCommit;
+
+  const silence = useVoiceSilenceCommit(
+    getSession,
+    () => muteAfterSilenceCommitRef.current(),
+    options.voiceStatus,
+  );
+  const silenceRef = useRef(silence);
+  silenceRef.current = silence;
+
+  const ensurePlayer = useCallback(async () => {
+    if (!playerRef.current) {
+      playerRef.current = new WsPcmPlayer();
+      playerRef.current.prewarm();
+    }
+    await playerRef.current.ensureRunning();
+  }, []);
+
+  const stopCapture = useCallback((commit: boolean) => {
+    const s = silenceRef.current;
+    s.clearPoll();
+    recorderRef.current?.stop();
+    if (!commit) return;
+    if (s.consumeSkipCommitOnMute()) return;
+    s.tryCommit('manual');
+  }, []);
+
+  const startCapture = useCallback(async () => {
+    const session = sessionRef.current;
+    if (!session) {
+      console.warn('[nxtlinq] startCapture skipped: voice session not bound');
+      return;
+    }
+    if (isIOS() && !iosSessionPrimedRef.current) {
+      try {
+        await waitForIOSAudioSessionReady();
+        iosSessionPrimedRef.current = true;
+      } catch (err) {
+        console.warn('[nxtlinq] waitForIOSAudioSessionReady failed', err);
+      }
+    }
+    const s = silenceRef.current;
+    if (!recorderRef.current) {
+      recorderRef.current = new WsPcmRecorder();
+    }
+    playerRef.current?.clearQueue();
+    recorderRef.current.bindSession(session);
+    recorderRef.current.setOnRms(s.onSpeechRms);
+    s.resetTurn();
+    try {
+      await recorderRef.current.initialize();
+      await recorderRef.current.start();
+      s.startPoll();
+    } catch (err) {
+      s.clearPoll();
+      console.error('[nxtlinq] mic capture start failed', err);
+    }
+  }, []);
+
+  const cleanup = useCallback(() => {
+    const s = silenceRef.current;
+    s.clearPoll();
+    iosSessionPrimedRef.current = false;
+    recorderRef.current?.stop();
+    recorderRef.current?.cleanup();
+    recorderRef.current = null;
+    playerRef.current?.cleanup();
+    playerRef.current = null;
+    sessionRef.current = null;
+  }, []);
+
+  const stopCaptureRef = useRef(stopCapture);
+  const startCaptureRef = useRef(startCapture);
+  const cleanupRef = useRef(cleanup);
+  stopCaptureRef.current = stopCapture;
+  startCaptureRef.current = startCapture;
+  cleanupRef.current = cleanup;
+
+  const bindSession = useCallback(
+    (session: VoiceSession | null, captureWhenUnmuted = false) => {
+      sessionRef.current = session;
+      recorderRef.current?.bindSession(session);
+      if (session && captureWhenUnmuted && isCaptureActiveRef.current) {
+        void startCaptureRef.current();
+      }
+    },
+    [],
+  );
+
+  const buildCallbacks = useCallback(
+    (overrides?: Partial<WsVoiceCallbacks>): WsVoiceCallbacks => ({
+      onOpen: () => {
+        void ensurePlayer();
+        overrides?.onOpen?.();
+      },
+      onAudioDelta: (pcm16) => {
+        void ensurePlayer().then(() => playerRef.current?.addAudio(pcm16));
+        overrides?.onAudioDelta?.(pcm16);
+      },
+      onClose: (reason) => {
+        cleanupRef.current();
+        overrides?.onClose?.(reason);
+      },
+      onError: (err) => {
+        cleanupRef.current();
+        overrides?.onError?.(err);
+      },
+    }),
+    [ensurePlayer],
+  );
+
+  useEffect(() => {
+    if (!isVoiceActive) {
+      prevCaptureActiveRef.current = false;
+      cleanupRef.current();
+      return;
+    }
+    const prev = prevCaptureActiveRef.current;
+    prevCaptureActiveRef.current = isCaptureActive;
+    if (isCaptureActive && !prev) {
+      void startCaptureRef.current().catch((err) => {
+        console.error('[nxtlinq] mic capture start failed', err);
+      });
+      return;
+    }
+    if (!isCaptureActive && prev) {
+      stopCaptureRef.current(true);
+    }
+  }, [isCaptureActive, isVoiceActive]);
+
+  useEffect(() => {
+    if (!isVoiceActive) return;
+    if (!recorderRef.current) {
+      recorderRef.current = new WsPcmRecorder();
+    }
+    void recorderRef.current.initialize().catch((err) => {
+      console.warn('[nxtlinq] WsPcmRecorder prewarm failed', err);
+    });
+  }, [isVoiceActive]);
+
+  useEffect(() => () => cleanupRef.current(), []);
+
+  const getOutputAudioLevel = useCallback(() => {
+    const fromPlayer = playerRef.current?.getAudioLevel() ?? 0;
+    if (fromPlayer > 0) return fromPlayer;
+    return sessionRef.current?.getOutputAudioLevel() ?? 0;
+  }, []);
+
+  return {
+    buildCallbacks,
+    bindSession,
+    cleanup,
+    beginCapture: startCapture,
+    endCapture: stopCapture,
+    getOutputAudioLevel,
+  };
+}

package/src/voice/voiceImagePickerOptions.ts

@@ -0,0 +1,10 @@
+/** Smaller payload for WS / SCTP voice channels (Berify-aligned). */
+export const VOICE_IMAGE_PICKER_OPTIONS = {
+  mediaType: 'photo' as const,
+  multiple: false,
+  compressImageMaxWidth: 384,
+  compressImageMaxHeight: 384,
+  compressImageQuality: 0.55,
+  /** iOS albums are often HEIC; force JPEG for OpenAI-compatible MIME. */
+  forceJpg: true,
+};

package/src/voice/voiceSilenceConstants.ts

@@ -0,0 +1,10 @@
+/** Min RMS to treat as user speech (~-42 dBFS). Berify-aligned. */
+export const MIC_SPEECH_RMS_THRESHOLD = 0.008;
+
+/** Louder threshold while assistant speaks — reduces echo false triggers. */
+export const MIC_BARGE_IN_RMS_THRESHOLD = 0.045;
+
+/** Commit turn after this much silence while mic is open. */
+export const MIC_SILENCE_COMMIT_MS = 400;
+
+export const MIC_SILENCE_POLL_MS = 100;