@bytexbyte/nxtlinq-ai-agent-ui-react-development 0.1.1 → 0.1.3

package/src/voice/useVoiceTranscriptMessages.ts

@@ -5,11 +5,13 @@ import type {
 } from '@bytexbyte/nxtlinq-ai-agent-core-development';
 import { mergeStreamingTranscript } from '@bytexbyte/nxtlinq-ai-agent-core-development';
 import { useCallback, useRef } from 'react';
-import type { InteractionMode } from '../context/AgentAssistantContext';
+import { ensureUserBubbleForVoiceTurn } from './voiceUserBubble';
 
-type VoiceTranscriptAgentApi = {
+type InteractionMode = 'text' | 'voice';
+
+export type VoiceTranscriptAgentApi = {
   getMessages: () => Message[];
-  setMessages: (messages: Message[]) => void;
+  updateMessages: (updater: (prev: Message[]) => Message[]) => void;
   syncVoiceTurnHistory: (options?: { last?: number }) => Promise<void>;
 };
 
@@ -26,6 +28,7 @@ export function useVoiceTranscriptMessages(
   api: VoiceTranscriptAgentApi,
   interactionMode: InteractionMode,
   voiceSessionId: string | null,
+  getPendingUserText?: () => string,
 ) {
   const streamIdRef = useRef<string | null>(null);
   const sessionIdRef = useRef(voiceSessionId);
@@ -38,42 +41,48 @@ export function useVoiceTranscriptMessages(
 
   const upsertStreaming = useCallback(
     (text: string) => {
-      const messages = api.getMessages();
-      let streamId = streamIdRef.current;
-      if (!streamId) {
-        streamId = `${STREAM_PREFIX}${Date.now()}`;
-        streamIdRef.current = streamId;
-      }
-      const idx = messages.findIndex((m) => m.id === streamId);
-      const partialContent =
-        idx >= 0
-          ? mergeStreamingTranscript(messages[idx]?.partialContent ?? '', text)
-          : text;
-      const meta = voiceMeta(sessionIdRef.current);
-      if (idx >= 0) {
-        api.setMessages(
-          messages.map((m, i) =>
+      api.updateMessages((prev) => {
+        let streamId = streamIdRef.current;
+        if (!streamId) {
+          streamId = `${STREAM_PREFIX}${Date.now()}`;
+          streamIdRef.current = streamId;
+        }
+        const idx = prev.findIndex((m) => m.id === streamId);
+        const partialContent =
+          idx >= 0
+            ? mergeStreamingTranscript(prev[idx]?.partialContent ?? '', text)
+            : text;
+        const meta = voiceMeta(sessionIdRef.current);
+
+        if (idx >= 0) {
+          return prev.map((m, i) =>
             i === idx
               ? { ...m, partialContent, isStreaming: true, metadata: { ...m.metadata, ...meta } }
               : m,
-          ),
+          );
+        }
+
+        const withUser = ensureUserBubbleForVoiceTurn(
+          prev,
+          getPendingUserText?.() ?? '',
+          undefined,
+          meta,
         );
-        return;
-      }
-      api.setMessages([
-        ...messages,
-        {
-          id: streamId,
-          role: 'assistant',
-          content: '',
-          partialContent,
-          isStreaming: true,
-          timestamp: new Date().toISOString(),
-          metadata: meta,
-        },
-      ]);
+        return [
+          ...withUser,
+          {
+            id: streamId,
+            role: 'assistant' as const,
+            content: '',
+            partialContent,
+            isStreaming: true,
+            timestamp: new Date().toISOString(),
+            metadata: meta,
+          },
+        ];
+      });
     },
-    [api],
+    [api, getPendingUserText],
   );
 
   const finalizeAssistant = useCallback(
@@ -82,11 +91,10 @@ export function useVoiceTranscriptMessages(
       streamIdRef.current = null;
       if (!trimmed) return;
 
-      const messages = api.getMessages();
-      const streamIdx = messages.findIndex((m) => m.isStreaming && m.role === 'assistant');
-      if (streamIdx >= 0) {
-        api.setMessages(
-          messages.map((m, i) =>
+      api.updateMessages((prev) => {
+        const streamIdx = prev.findIndex((m) => m.isStreaming && m.role === 'assistant');
+        if (streamIdx >= 0) {
+          return prev.map((m, i) =>
             i === streamIdx
               ? {
                   ...m,
@@ -97,22 +105,21 @@ export function useVoiceTranscriptMessages(
                   metadata: { ...m.metadata, ...voiceMeta(sessionIdRef.current) },
                 }
               : m,
-          ),
-        );
-        return;
-      }
-      const last = messages[messages.length - 1];
-      if (last?.role === 'assistant' && last.content === trimmed) return;
-      api.setMessages([
-        ...messages,
-        {
-          id: messageId ?? `voice-asst-${Date.now()}`,
-          role: 'assistant',
-          content: trimmed,
-          timestamp: new Date().toISOString(),
-          metadata: voiceMeta(sessionIdRef.current),
-        },
-      ]);
+          );
+        }
+        const last = prev[prev.length - 1];
+        if (last?.role === 'assistant' && last.content === trimmed) return prev;
+        return [
+          ...prev,
+          {
+            id: messageId ?? `voice-asst-${Date.now()}`,
+            role: 'assistant' as const,
+            content: trimmed,
+            timestamp: new Date().toISOString(),
+            metadata: voiceMeta(sessionIdRef.current),
+          },
+        ];
+      });
     },
     [api],
   );
@@ -122,31 +129,31 @@ export function useVoiceTranscriptMessages(
       if (!isVoiceUiActive()) return;
       const text = event.text?.trim() ?? '';
       if (event.role === 'assistant') {
-        // Keep one streaming bubble for the whole turn; finalize only in handleDone.
         if (text) upsertStreaming(text);
         return;
       }
       if (event.role === 'user' && !event.interim && text) {
-        const messages = api.getMessages();
-        const last = messages[messages.length - 1];
-        if (last?.role === 'user' && last.content === text) return;
-        api.setMessages([
-          ...messages,
-          {
-            id: `voice-user-${Date.now()}`,
-            role: 'user',
-            content: text,
-            timestamp: new Date().toISOString(),
-            metadata: voiceMeta(sessionIdRef.current),
-          },
-        ]);
+        api.updateMessages((prev) => {
+          const last = prev[prev.length - 1];
+          if (last?.role === 'user' && last.content === text) return prev;
+          return [
+            ...prev,
+            {
+              id: `voice-user-${Date.now()}`,
+              role: 'user' as const,
+              content: text,
+              timestamp: new Date().toISOString(),
+              metadata: voiceMeta(sessionIdRef.current),
+            },
+          ];
+        });
       }
     },
-    [api, finalizeAssistant, isVoiceUiActive, upsertStreaming],
+    [api, isVoiceUiActive, upsertStreaming],
   );
 
   const handleDone = useCallback(
-    (event: VoiceDoneEvent) => {
+    (event: VoiceDoneEvent, options?: { pendingUserText?: string }) => {
       if (!isVoiceUiActive()) return;
       if (event.guardrailsBlocked || event.billingBlocked || event.error) {
         streamIdRef.current = null;
@@ -154,6 +161,14 @@ export function useVoiceTranscriptMessages(
       }
       const reply = event.replyText?.trim() ?? '';
       if (reply) {
+        api.updateMessages((prev) =>
+          ensureUserBubbleForVoiceTurn(
+            prev,
+            options?.pendingUserText ?? getPendingUserText?.() ?? '',
+            event.userMessageId,
+            voiceMeta(sessionIdRef.current),
+          ),
+        );
         finalizeAssistant(reply, event.assistantMessageId ?? undefined);
       } else {
         streamIdRef.current = null;
@@ -162,7 +177,7 @@ export function useVoiceTranscriptMessages(
         console.warn('[nxtlinq] syncVoiceTurnHistory after voice turn failed', err);
       });
     },
-    [api, finalizeAssistant, isVoiceUiActive],
+    [api, finalizeAssistant, getPendingUserText, isVoiceUiActive],
   );
 
   const clearVoiceStream = useCallback(() => {

package/src/voice/voiceUserBubble.ts

@@ -0,0 +1,71 @@
+import type { Message } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export const VOICE_USER_INPUT_PLACEHOLDER = '(Voice input)';
+
+type VoiceMeta = {
+  voiceRealtime: true;
+  voiceSessionId?: string;
+};
+
+/** Ensure a user bubble exists before the in-flight assistant reply for this turn. */
+export function ensureUserBubbleForVoiceTurn(
+  messages: Message[],
+  userText: string,
+  userMessageId: string | null | undefined,
+  metaForPlaceholder: VoiceMeta | undefined,
+): Message[] {
+  const streamIdx = messages.findIndex((m) => m.isStreaming && m.role === 'assistant');
+  const insertAt = streamIdx >= 0 ? streamIdx : messages.length;
+  const before = messages.slice(0, insertAt);
+
+  const lastAsstIdx = (() => {
+    for (let i = before.length - 1; i >= 0; i -= 1) {
+      const m = before[i];
+      if (m.role === 'assistant' && !m.isStreaming && Boolean(m.content?.trim())) return i;
+    }
+    return -1;
+  })();
+  const lastUserIdx = (() => {
+    for (let i = before.length - 1; i >= 0; i -= 1) {
+      if (before[i].role === 'user') return i;
+    }
+    return -1;
+  })();
+  const hasUserForTurn = lastUserIdx >= 0 && lastUserIdx > lastAsstIdx;
+
+  const trimmed = userText.trim();
+  const displayText = trimmed || VOICE_USER_INPUT_PLACEHOLDER;
+
+  if (hasUserForTurn) {
+    const existing = before[lastUserIdx];
+    const shouldUpgrade =
+      trimmed &&
+      existing.content !== trimmed &&
+      (existing.content === VOICE_USER_INPUT_PLACEHOLDER || !existing.content.trim());
+    if (!shouldUpgrade) return messages;
+
+    const upgraded = before.map((m, i) =>
+      i === lastUserIdx
+        ? {
+            ...m,
+            content: trimmed,
+            id: userMessageId ?? m.id,
+            metadata: userMessageId ? undefined : m.metadata,
+          }
+        : m,
+    );
+    return [...upgraded, ...messages.slice(insertAt)];
+  }
+
+  const userMsg: Message = {
+    id: userMessageId ?? `voice-user-${Date.now()}`,
+    role: 'user',
+    content: displayText,
+    timestamp: new Date().toISOString(),
+    metadata: userMessageId ? undefined : metaForPlaceholder,
+  };
+
+  const next = [...messages];
+  next.splice(insertAt, 0, userMsg);
+  return next;
+}