@bytexbyte/nxtlinq-ai-agent-ui-react-development 0.1.2 → 0.1.4

package/src/voice/useVoiceConnectOrchestration.ts

@@ -1,117 +0,0 @@
-import type { VoiceDoneEvent, VoiceTranscriptEvent } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-import type {
-  UseNxtlinqAgentResult,
-  UseNxtlinqVoiceResult,
-} from '@bytexbyte/nxtlinq-ai-agent-web-development';
-import { useCallback, useRef } from 'react';
-import type { InteractionMode } from '../context/AgentAssistantContext';
-
-type VoiceCallbacks = {
-  handleTranscript: (event: VoiceTranscriptEvent) => void;
-  handleDone: (event: VoiceDoneEvent) => void;
-  clearVoiceStream: () => void;
-  prepareForVoiceConnect: () => void;
-  resetMicState: () => void;
-  clearAssistantMicHold: () => void;
-  setInteractionMode: (mode: InteractionMode) => void;
-  setIsVoiceConnecting: (connecting: boolean) => void;
-};
-
-export function useVoiceConnectOrchestration(
-  agent: UseNxtlinqAgentResult,
-  voice: UseNxtlinqVoiceResult,
-  interactionMode: InteractionMode,
-  micStartsMuted: boolean,
-  callbacks: VoiceCallbacks,
-) {
-  const voiceConnectChainRef = useRef<Promise<unknown>>(Promise.resolve());
-
-  const wrappedStartVoice = useCallback(
-    (options?: Parameters<typeof voice.startVoice>[0]) => {
-      const connect = async () => {
-        if (voice.voiceSessionId != null) {
-          await voice.stopVoice('restart_voice');
-        }
-        callbacks.prepareForVoiceConnect();
-        callbacks.setInteractionMode('voice');
-        callbacks.setIsVoiceConnecting(true);
-        try {
-          const session = await voice.startVoice({
-            startWithMicMuted: micStartsMuted,
-            keepMicCaptureActive: true,
-            ...options,
-            onOpen: () => {
-              if (!micStartsMuted) {
-                voice.muteMic(false);
-              }
-              options?.onOpen?.();
-            },
-            onClose: (reason) => {
-              callbacks.clearVoiceStream();
-              callbacks.resetMicState();
-              const userInitiated =
-                reason === 'switch_to_text' ||
-                reason === 'client_stop' ||
-                reason === 'mode_text_cleanup';
-              if (userInitiated) {
-                callbacks.setInteractionMode('text');
-              } else {
-                console.warn('[nxtlinq] voice session closed:', reason);
-              }
-              options?.onClose?.(reason);
-            },
-            onError: (err) => {
-              callbacks.clearVoiceStream();
-              callbacks.resetMicState();
-              console.warn('[nxtlinq] voice session error:', err.message);
-              options?.onError?.(err);
-            },
-            onTranscript: (event) => {
-              callbacks.handleTranscript(event);
-              options?.onTranscript?.(event);
-            },
-            onDone: (event) => {
-              callbacks.handleDone(event);
-              options?.onDone?.(event);
-            },
-          });
-          return session;
-        } catch (err) {
-          callbacks.setInteractionMode('text');
-          callbacks.resetMicState();
-          throw err;
-        } finally {
-          callbacks.setIsVoiceConnecting(false);
-        }
-      };
-      const next = voiceConnectChainRef.current.then(connect, connect);
-      voiceConnectChainRef.current = next.catch(() => undefined);
-      return next;
-    },
-    [voice, micStartsMuted, callbacks],
-  );
-
-  const wrappedStopVoice = useCallback(async () => {
-    await voice.stopVoice('client_stop');
-    callbacks.resetMicState();
-    callbacks.setInteractionMode('text');
-  }, [voice, callbacks]);
-
-  const wrappedInterrupt = useCallback(() => {
-    voice.interrupt();
-    callbacks.clearAssistantMicHold();
-  }, [voice, callbacks]);
-
-  const isVoiceChannelReady =
-    interactionMode === 'voice' &&
-    voice.voiceSessionId != null &&
-    (Boolean(agent.agent.getVoiceSession()?.isAppChannelOpen()) ||
-      voice.voiceStatus === 'listening');
-
-  return {
-    wrappedStartVoice,
-    wrappedStopVoice,
-    wrappedInterrupt,
-    isVoiceChannelReady,
-  };
-}

package/src/voice/useVoiceMicState.ts

@@ -1,117 +0,0 @@
-import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-import type { UseNxtlinqVoiceResult } from '@bytexbyte/nxtlinq-ai-agent-web-development';
-import { useCallback, useEffect, useRef, useState } from 'react';
-import { ASSISTANT_MIC_HOLD_STATUSES } from './voiceMicConstants';
-
-export type UseVoiceMicStateOptions = {
-  startWithMicMuted?: boolean;
-  holdMicDuringAssistant?: boolean;
-  onBargeIn?: () => void;
-};
-
-export function useVoiceMicState(
-  voice: UseNxtlinqVoiceResult,
-  isVoiceConnecting: boolean,
-  options?: UseVoiceMicStateOptions,
-) {
-  const connectMuted = options?.startWithMicMuted !== false;
-  const holdDuringAssistant = options?.holdMicDuringAssistant !== false;
-  const userMicMutedRef = useRef(connectMuted);
-  const assistantMicHoldRef = useRef(false);
-  const userMicOptInRef = useRef(!connectMuted);
-  const [isMicMuted, setIsMicMuted] = useState(connectMuted);
-
-  const applyMicState = useCallback(() => {
-    const shouldMute = userMicMutedRef.current || assistantMicHoldRef.current;
-    voice.muteMic(shouldMute);
-    setIsMicMuted(shouldMute);
-  }, [voice]);
-
-  const resetMicState = useCallback(() => {
-    userMicMutedRef.current = false;
-    assistantMicHoldRef.current = false;
-    userMicOptInRef.current = false;
-    setIsMicMuted(false);
-  }, []);
-
-  const prepareForVoiceConnect = useCallback(() => {
-    userMicMutedRef.current = connectMuted;
-    userMicOptInRef.current = !connectMuted;
-    assistantMicHoldRef.current = false;
-    setIsMicMuted(connectMuted);
-    voice.muteMic(connectMuted);
-  }, [voice, connectMuted]);
-
-  useEffect(() => {
-    if (!isVoiceConnecting) return;
-    userMicMutedRef.current = connectMuted;
-    voice.muteMic(connectMuted);
-    setIsMicMuted(connectMuted);
-  }, [isVoiceConnecting, voice, connectMuted]);
-
-  const prevVoiceStatusRef = useRef(voice.voiceStatus);
-
-  useEffect(() => {
-    const status = voice.voiceStatus;
-    const prev = prevVoiceStatusRef.current;
-    prevVoiceStatusRef.current = status;
-
-    if (holdDuringAssistant && ASSISTANT_MIC_HOLD_STATUSES.has(status)) {
-      if (!userMicMutedRef.current && userMicOptInRef.current) {
-        assistantMicHoldRef.current = false;
-        applyMicState();
-        return;
-      }
-      assistantMicHoldRef.current = true;
-      applyMicState();
-      return;
-    }
-    if (status === 'listening' || status === 'idle') {
-      assistantMicHoldRef.current = false;
-      if (
-        connectMuted
-        && status === 'listening'
-        && prev === 'speaking'
-        && !userMicOptInRef.current
-      ) {
-        userMicMutedRef.current = true;
-      }
-      applyMicState();
-    }
-  }, [voice.voiceStatus, applyMicState, holdDuringAssistant, connectMuted]);
-
-  const toggleVoiceMicMute = useCallback(() => {
-    if (!voice.isVoiceActive && !isVoiceConnecting) return;
-    if (assistantMicHoldRef.current && userMicMutedRef.current) {
-      assistantMicHoldRef.current = false;
-      userMicMutedRef.current = false;
-      userMicOptInRef.current = true;
-      options?.onBargeIn?.();
-      applyMicState();
-      return;
-    }
-    if (assistantMicHoldRef.current) {
-      userMicMutedRef.current = true;
-      applyMicState();
-      return;
-    }
-    const nextMuted = !userMicMutedRef.current;
-    userMicMutedRef.current = nextMuted;
-    userMicOptInRef.current = !nextMuted;
-    applyMicState();
-  }, [voice.isVoiceActive, isVoiceConnecting, applyMicState, options]);
-
-  const clearAssistantMicHold = useCallback(() => {
-    assistantMicHoldRef.current = false;
-    applyMicState();
-  }, [applyMicState]);
-
-  return {
-    isMicMuted,
-    isMicHeldForAssistant: ASSISTANT_MIC_HOLD_STATUSES.has(voice.voiceStatus),
-    toggleVoiceMicMute,
-    prepareForVoiceConnect,
-    resetMicState,
-    clearAssistantMicHold,
-  };
-}

package/src/voice/useVoiceTranscriptMessages.ts

@@ -1,188 +0,0 @@
-import type {
-  Message,
-  VoiceDoneEvent,
-  VoiceTranscriptEvent,
-} from '@bytexbyte/nxtlinq-ai-agent-core-development';
-import { mergeStreamingTranscript } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-import { useCallback, useRef } from 'react';
-import { ensureUserBubbleForVoiceTurn } from './voiceUserBubble';
-
-type InteractionMode = 'text' | 'voice';
-
-export type VoiceTranscriptAgentApi = {
-  getMessages: () => Message[];
-  updateMessages: (updater: (prev: Message[]) => Message[]) => void;
-  syncVoiceTurnHistory: (options?: { last?: number }) => Promise<void>;
-};
-
-const STREAM_PREFIX = 'voice-stream-';
-
-function voiceMeta(sessionId: string | null) {
-  return {
-    voiceRealtime: true as const,
-    voiceSessionId: sessionId ?? undefined,
-  };
-}
-
-export function useVoiceTranscriptMessages(
-  api: VoiceTranscriptAgentApi,
-  interactionMode: InteractionMode,
-  voiceSessionId: string | null,
-  getPendingUserText?: () => string,
-) {
-  const streamIdRef = useRef<string | null>(null);
-  const sessionIdRef = useRef(voiceSessionId);
-  sessionIdRef.current = voiceSessionId;
-
-  const isVoiceUiActive = useCallback(
-    () => interactionMode === 'voice' && sessionIdRef.current != null,
-    [interactionMode],
-  );
-
-  const upsertStreaming = useCallback(
-    (text: string) => {
-      api.updateMessages((prev) => {
-        let streamId = streamIdRef.current;
-        if (!streamId) {
-          streamId = `${STREAM_PREFIX}${Date.now()}`;
-          streamIdRef.current = streamId;
-        }
-        const idx = prev.findIndex((m) => m.id === streamId);
-        const partialContent =
-          idx >= 0
-            ? mergeStreamingTranscript(prev[idx]?.partialContent ?? '', text)
-            : text;
-        const meta = voiceMeta(sessionIdRef.current);
-
-        if (idx >= 0) {
-          return prev.map((m, i) =>
-            i === idx
-              ? { ...m, partialContent, isStreaming: true, metadata: { ...m.metadata, ...meta } }
-              : m,
-          );
-        }
-
-        const withUser = ensureUserBubbleForVoiceTurn(
-          prev,
-          getPendingUserText?.() ?? '',
-          undefined,
-          meta,
-        );
-        return [
-          ...withUser,
-          {
-            id: streamId,
-            role: 'assistant' as const,
-            content: '',
-            partialContent,
-            isStreaming: true,
-            timestamp: new Date().toISOString(),
-            metadata: meta,
-          },
-        ];
-      });
-    },
-    [api, getPendingUserText],
-  );
-
-  const finalizeAssistant = useCallback(
-    (text: string, messageId?: string | null) => {
-      const trimmed = text.trim();
-      streamIdRef.current = null;
-      if (!trimmed) return;
-
-      api.updateMessages((prev) => {
-        const streamIdx = prev.findIndex((m) => m.isStreaming && m.role === 'assistant');
-        if (streamIdx >= 0) {
-          return prev.map((m, i) =>
-            i === streamIdx
-              ? {
-                  ...m,
-                  id: messageId ?? m.id,
-                  content: trimmed,
-                  partialContent: undefined,
-                  isStreaming: false,
-                  metadata: { ...m.metadata, ...voiceMeta(sessionIdRef.current) },
-                }
-              : m,
-          );
-        }
-        const last = prev[prev.length - 1];
-        if (last?.role === 'assistant' && last.content === trimmed) return prev;
-        return [
-          ...prev,
-          {
-            id: messageId ?? `voice-asst-${Date.now()}`,
-            role: 'assistant' as const,
-            content: trimmed,
-            timestamp: new Date().toISOString(),
-            metadata: voiceMeta(sessionIdRef.current),
-          },
-        ];
-      });
-    },
-    [api],
-  );
-
-  const handleTranscript = useCallback(
-    (event: VoiceTranscriptEvent) => {
-      if (!isVoiceUiActive()) return;
-      const text = event.text?.trim() ?? '';
-      if (event.role === 'assistant') {
-        if (text) upsertStreaming(text);
-        return;
-      }
-      if (event.role === 'user' && !event.interim && text) {
-        api.updateMessages((prev) => {
-          const last = prev[prev.length - 1];
-          if (last?.role === 'user' && last.content === text) return prev;
-          return [
-            ...prev,
-            {
-              id: `voice-user-${Date.now()}`,
-              role: 'user' as const,
-              content: text,
-              timestamp: new Date().toISOString(),
-              metadata: voiceMeta(sessionIdRef.current),
-            },
-          ];
-        });
-      }
-    },
-    [api, isVoiceUiActive, upsertStreaming],
-  );
-
-  const handleDone = useCallback(
-    (event: VoiceDoneEvent, options?: { pendingUserText?: string }) => {
-      if (!isVoiceUiActive()) return;
-      if (event.guardrailsBlocked || event.billingBlocked || event.error) {
-        streamIdRef.current = null;
-        return;
-      }
-      const reply = event.replyText?.trim() ?? '';
-      if (reply) {
-        api.updateMessages((prev) =>
-          ensureUserBubbleForVoiceTurn(
-            prev,
-            options?.pendingUserText ?? getPendingUserText?.() ?? '',
-            event.userMessageId,
-            voiceMeta(sessionIdRef.current),
-          ),
-        );
-        finalizeAssistant(reply, event.assistantMessageId ?? undefined);
-      } else {
-        streamIdRef.current = null;
-      }
-      void api.syncVoiceTurnHistory({ last: 20 }).catch((err) => {
-        console.warn('[nxtlinq] syncVoiceTurnHistory after voice turn failed', err);
-      });
-    },
-    [api, finalizeAssistant, getPendingUserText, isVoiceUiActive],
-  );
-
-  const clearVoiceStream = useCallback(() => {
-    streamIdRef.current = null;
-  }, []);
-
-  return { handleTranscript, handleDone, clearVoiceStream };
-}

package/src/voice/voiceMicConstants.ts

@@ -1,13 +0,0 @@
-import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-
-export const ASSISTANT_MIC_HOLD_STATUSES: ReadonlySet<VoiceStatus> = new Set([
-  'transcribing',
-  'thinking',
-  'generating',
-  'speaking',
-]);
-
-export const SPEAKER_ACTIVE_STATUSES: ReadonlySet<VoiceStatus> = new Set([
-  'generating',
-  'speaking',
-]);

package/src/voice/voiceUserBubble.ts

@@ -1,71 +0,0 @@
-import type { Message } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-
-export const VOICE_USER_INPUT_PLACEHOLDER = '(Voice input)';
-
-type VoiceMeta = {
-  voiceRealtime: true;
-  voiceSessionId?: string;
-};
-
-/** Ensure a user bubble exists before the in-flight assistant reply for this turn. */
-export function ensureUserBubbleForVoiceTurn(
-  messages: Message[],
-  userText: string,
-  userMessageId: string | null | undefined,
-  metaForPlaceholder: VoiceMeta | undefined,
-): Message[] {
-  const streamIdx = messages.findIndex((m) => m.isStreaming && m.role === 'assistant');
-  const insertAt = streamIdx >= 0 ? streamIdx : messages.length;
-  const before = messages.slice(0, insertAt);
-
-  const lastAsstIdx = (() => {
-    for (let i = before.length - 1; i >= 0; i -= 1) {
-      const m = before[i];
-      if (m.role === 'assistant' && !m.isStreaming && Boolean(m.content?.trim())) return i;
-    }
-    return -1;
-  })();
-  const lastUserIdx = (() => {
-    for (let i = before.length - 1; i >= 0; i -= 1) {
-      if (before[i].role === 'user') return i;
-    }
-    return -1;
-  })();
-  const hasUserForTurn = lastUserIdx >= 0 && lastUserIdx > lastAsstIdx;
-
-  const trimmed = userText.trim();
-  const displayText = trimmed || VOICE_USER_INPUT_PLACEHOLDER;
-
-  if (hasUserForTurn) {
-    const existing = before[lastUserIdx];
-    const shouldUpgrade =
-      trimmed &&
-      existing.content !== trimmed &&
-      (existing.content === VOICE_USER_INPUT_PLACEHOLDER || !existing.content.trim());
-    if (!shouldUpgrade) return messages;
-
-    const upgraded = before.map((m, i) =>
-      i === lastUserIdx
-        ? {
-            ...m,
-            content: trimmed,
-            id: userMessageId ?? m.id,
-            metadata: userMessageId ? undefined : m.metadata,
-          }
-        : m,
-    );
-    return [...upgraded, ...messages.slice(insertAt)];
-  }
-
-  const userMsg: Message = {
-    id: userMessageId ?? `voice-user-${Date.now()}`,
-    role: 'user',
-    content: displayText,
-    timestamp: new Date().toISOString(),
-    metadata: userMessageId ? undefined : metaForPlaceholder,
-  };
-
-  const next = [...messages];
-  next.splice(insertAt, 0, userMsg);
-  return next;
-}