@agent-native/core 0.94.1 → 0.94.3

package/corpus/core/src/client/composer/useRealtimeVoiceMode.tsx

@@ -10,6 +10,7 @@ import {
 } from "react";
 import { createPortal } from "react-dom";
 
+import { requestAgentChatThreadOpen } from "../agent-chat.js";
 import {
   SIDEBAR_STATE_CHANGE_EVENT,
   type AgentSidebarStateChangeDetail,
@@ -18,6 +19,13 @@ import { agentNativePath } from "../api-path.js";
 import { readClientAppState, setClientAppState } from "../application-state.js";
 import { getBrowserTabId } from "../browser-tab-id.js";
 import { useT } from "../i18n.js";
+import {
+  createRealtimeVoiceAudioLevelStore,
+  normalizeRealtimeVoiceRms,
+  smoothRealtimeVoiceLevel,
+  type RealtimeVoiceAudioLevelStore,
+} from "./realtime-voice-audio-level.js";
+import { realtimeVoiceTranscriptRegistry } from "./realtime-voice-transcript.js";
 import {
   RealtimeVoiceModeDock,
   type RealtimeVoiceModeCopy,
@@ -43,11 +51,54 @@ export interface RealtimeVoiceModeApi {
   active: boolean;
   errorMessage: string | null;
   chatVisible: boolean;
+  audioLevels: RealtimeVoiceAudioLevelStore;
   start: () => Promise<void>;
   end: () => void;
   toggleChat: () => void;
 }
 
+export interface CompletedRealtimeVoiceTranscript {
+  role: "user" | "assistant";
+  text: string;
+  providerId?: string;
+}
+
+/**
+ * Voice mode owns the chat only temporarily. Restore the captured transcript
+ * when it is still the user's active thread (or the chat has no active thread)
+ * but never pull them back after they deliberately selected another thread.
+ */
+export function shouldRestoreRealtimeVoiceTranscriptThread(
+  transcriptThreadId: string | undefined,
+  activeThreadId: string | undefined,
+): transcriptThreadId is string {
+  return Boolean(
+    transcriptThreadId &&
+    (!activeThreadId || activeThreadId === transcriptThreadId),
+  );
+}
+
+export function extractCompletedRealtimeVoiceTranscript(
+  event: RealtimeServerEvent,
+): CompletedRealtimeVoiceTranscript | null {
+  const userCompleted =
+    event.type === "conversation.item.input_audio_transcription.completed";
+  const assistantCompleted =
+    event.type === "response.output_audio_transcript.done";
+  if (!userCompleted && !assistantCompleted) return null;
+  const text =
+    typeof event.transcript === "string" ? event.transcript.trim() : "";
+  if (!text) return null;
+  const providerId = [event.item_id, event.response_id, event.event_id].find(
+    (value): value is string => typeof value === "string" && value.length > 0,
+  );
+  return {
+    role: userCompleted ? "user" : "assistant",
+    text,
+    ...(providerId ? { providerId } : {}),
+  };
+}
+
 export interface RealtimeVoiceModeProviderProps {
   children: ReactNode;
   browserTabId?: string;
@@ -232,17 +283,29 @@ function useRealtimeVoiceModeController(
   const [state, setState] = useState<"idle" | RealtimeVoiceModeState>("idle");
   const [error, setError] = useState<string | null>(null);
   const [chatVisible, setChatVisible] = useState(false);
+  const [audioLevels] = useState(createRealtimeVoiceAudioLevelStore);
   const stateRef = useRef(state);
   const peerRef = useRef<RTCPeerConnection | null>(null);
   const channelRef = useRef<RTCDataChannel | null>(null);
   const streamRef = useRef<MediaStream | null>(null);
   const audioRef = useRef<HTMLAudioElement | null>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const inputAnalyserRef = useRef<AnalyserNode | null>(null);
+  const outputAnalyserRef = useRef<AnalyserNode | null>(null);
+  const inputMeterBufferRef = useRef<Uint8Array<ArrayBuffer> | null>(null);
+  const outputMeterBufferRef = useRef<Uint8Array<ArrayBuffer> | null>(null);
+  const inputSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
+  const outputSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
+  const meterFrameRef = useRef<number | null>(null);
+  const lastMeterSampleRef = useRef(0);
   const abortRef = useRef<AbortController | null>(null);
   const handledCallsRef = useRef(new Set<string>());
   const sessionIdRef = useRef<string | undefined>(undefined);
   const startedAtRef = useRef<string | undefined>(undefined);
   const lastUserTextRef = useRef("");
   const lastAssistantTextRef = useRef("");
+  const transcriptThreadIdRef = useRef<string | undefined>(undefined);
+  const transcriptSequenceRef = useRef(0);
 
   useEffect(() => {
     stateRef.current = state;
@@ -279,6 +342,77 @@ function useRealtimeVoiceModeController(
     [syncAppState],
   );
 
+  const startMeterLoop = useCallback(() => {
+    if (meterFrameRef.current !== null) return;
+    const sample = (timestamp: number) => {
+      meterFrameRef.current = requestAnimationFrame(sample);
+      if (timestamp - lastMeterSampleRef.current < 50) return;
+      lastMeterSampleRef.current = timestamp;
+
+      const current = audioLevels.getSnapshot();
+      let input = current.input;
+      let output = current.output;
+      const inputAnalyser = inputAnalyserRef.current;
+      const inputBuffer = inputMeterBufferRef.current;
+      if (inputAnalyser && inputBuffer) {
+        inputAnalyser.getByteTimeDomainData(inputBuffer);
+        input = smoothRealtimeVoiceLevel(
+          input,
+          normalizeRealtimeVoiceRms(inputBuffer),
+        );
+      }
+      const outputAnalyser = outputAnalyserRef.current;
+      const outputBuffer = outputMeterBufferRef.current;
+      if (outputAnalyser && outputBuffer) {
+        outputAnalyser.getByteTimeDomainData(outputBuffer);
+        output = smoothRealtimeVoiceLevel(
+          output,
+          normalizeRealtimeVoiceRms(outputBuffer),
+        );
+      }
+      audioLevels.set({ input, output });
+    };
+    meterFrameRef.current = requestAnimationFrame(sample);
+  }, [audioLevels]);
+
+  const attachAudioMeter = useCallback(
+    (stream: MediaStream, channel: "input" | "output") => {
+      try {
+        const AudioCtor =
+          window.AudioContext ??
+          (
+            window as typeof window & {
+              webkitAudioContext?: typeof AudioContext;
+            }
+          ).webkitAudioContext;
+        if (!AudioCtor) return;
+        const context = audioContextRef.current ?? new AudioCtor();
+        audioContextRef.current = context;
+        void context.resume().catch(() => undefined);
+        const analyser = context.createAnalyser();
+        analyser.fftSize = 512;
+        const source = context.createMediaStreamSource(stream);
+        source.connect(analyser);
+        const buffer = new Uint8Array(analyser.frequencyBinCount);
+        if (channel === "input") {
+          inputSourceRef.current?.disconnect();
+          inputSourceRef.current = source;
+          inputAnalyserRef.current = analyser;
+          inputMeterBufferRef.current = buffer;
+        } else {
+          outputSourceRef.current?.disconnect();
+          outputSourceRef.current = source;
+          outputAnalyserRef.current = analyser;
+          outputMeterBufferRef.current = buffer;
+        }
+        startMeterLoop();
+      } catch {
+        // Audio metering is visual-only; keep the realtime call healthy.
+      }
+    },
+    [startMeterLoop],
+  );
+
   const cleanupTransport = useCallback(() => {
     abortRef.current?.abort();
     abortRef.current = null;
@@ -293,8 +427,25 @@ function useRealtimeVoiceModeController(
       audioRef.current.srcObject = null;
     }
     audioRef.current = null;
+    if (meterFrameRef.current !== null) {
+      cancelAnimationFrame(meterFrameRef.current);
+      meterFrameRef.current = null;
+    }
+    inputSourceRef.current?.disconnect();
+    outputSourceRef.current?.disconnect();
+    inputSourceRef.current = null;
+    outputSourceRef.current = null;
+    inputAnalyserRef.current = null;
+    outputAnalyserRef.current = null;
+    inputMeterBufferRef.current = null;
+    outputMeterBufferRef.current = null;
+    lastMeterSampleRef.current = 0;
+    const audioContext = audioContextRef.current;
+    audioContextRef.current = null;
+    if (audioContext) void audioContext.close().catch(() => undefined);
+    audioLevels.reset();
     handledCallsRef.current.clear();
-  }, []);
+  }, [audioLevels]);
 
   const fail = useCallback(
     (message: string, options?: { openKeySettings?: boolean }) => {
@@ -342,6 +493,26 @@ function useRealtimeVoiceModeController(
     [browserTabId, transition],
   );
 
+  const persistCompletedTranscript = useCallback(
+    (event: RealtimeServerEvent) => {
+      const transcript = extractCompletedRealtimeVoiceTranscript(event);
+      const threadId = transcriptThreadIdRef.current;
+      if (!transcript || !threadId) return;
+      const sessionIdentity =
+        sessionIdRef.current ?? startedAtRef.current ?? "pending";
+      const providerIdentity =
+        transcript.providerId ?? `sequence-${++transcriptSequenceRef.current}`;
+      realtimeVoiceTranscriptRegistry.publish({
+        id: `realtime-voice:${sessionIdentity}:${transcript.role}:${providerIdentity}`,
+        threadId,
+        role: transcript.role,
+        text: transcript.text,
+        createdAt: new Date().toISOString(),
+      });
+    },
+    [],
+  );
+
   const handleServerEvent = useCallback(
     (event: RealtimeServerEvent) => {
       if (event.type === "session.created") {
@@ -367,14 +538,16 @@ function useRealtimeVoiceModeController(
         if (typeof event.transcript === "string") {
           lastAssistantTextRef.current = event.transcript;
         }
+        persistCompletedTranscript(event);
         syncAppState("speaking");
       } else if (
         event.type === "conversation.item.input_audio_transcription.completed"
       ) {
         if (typeof event.transcript === "string") {
           lastUserTextRef.current = event.transcript;
-          syncAppState("working");
         }
+        persistCompletedTranscript(event);
+        syncAppState("working");
       } else if (event.type === "response.done") {
         const response = event.response;
         const status =
@@ -410,7 +583,14 @@ function useRealtimeVoiceModeController(
         transition("listening");
       }
     },
-    [copy, fail, handleFunctionCall, syncAppState, transition],
+    [
+      copy,
+      fail,
+      handleFunctionCall,
+      persistCompletedTranscript,
+      syncAppState,
+      transition,
+    ],
   );
 
   const start = useCallback(async () => {
@@ -430,6 +610,9 @@ function useRealtimeVoiceModeController(
     lastUserTextRef.current = "";
     lastAssistantTextRef.current = "";
     sessionIdRef.current = undefined;
+    transcriptThreadIdRef.current =
+      realtimeVoiceTranscriptRegistry.activeThreadId();
+    transcriptSequenceRef.current = 0;
     transition("connecting");
     setChatVisible(false);
     window.dispatchEvent(new Event("agent-panel:close"));
@@ -445,6 +628,7 @@ function useRealtimeVoiceModeController(
         },
       });
       streamRef.current = stream;
+      attachAudioMeter(stream, "input");
 
       const peer = new RTCPeerConnection();
       peerRef.current = peer;
@@ -455,7 +639,9 @@ function useRealtimeVoiceModeController(
       audio.setAttribute("playsinline", "");
       audioRef.current = audio;
       peer.ontrack = (trackEvent) => {
-        audio.srcObject = trackEvent.streams[0] ?? null;
+        const remoteStream = trackEvent.streams[0] ?? null;
+        audio.srcObject = remoteStream;
+        if (remoteStream) attachAudioMeter(remoteStream, "output");
         void audio.play().catch(() => undefined);
       };
 
@@ -501,15 +687,42 @@ function useRealtimeVoiceModeController(
       const status = (startError as { status?: unknown })?.status;
       fail(errorMessage(startError), { openKeySettings: status === 400 });
     }
-  }, [browserTabId, copy, fail, handleServerEvent, transition]);
+  }, [
+    attachAudioMeter,
+    browserTabId,
+    copy,
+    fail,
+    handleServerEvent,
+    transition,
+  ]);
 
   const end = useCallback(() => {
     if (stateRef.current === "idle" || stateRef.current === "ending") return;
+    const transcriptThreadId = transcriptThreadIdRef.current;
+    const activeThreadId = realtimeVoiceTranscriptRegistry.activeThreadId();
     transition("ending");
     cleanupTransport();
     setError(null);
     sessionIdRef.current = undefined;
     startedAtRef.current = undefined;
+    transcriptThreadIdRef.current = undefined;
+    setChatVisible(true);
+    if (
+      shouldRestoreRealtimeVoiceTranscriptThread(
+        transcriptThreadId,
+        activeThreadId,
+      )
+    ) {
+      requestAgentChatThreadOpen({
+        threadId: transcriptThreadId,
+        // The request is delivered asynchronously. Re-checking this at the
+        // receiver prevents a navigation that happened during that gap from
+        // being overwritten.
+        onlyIfActiveThreadId: transcriptThreadId,
+      });
+    } else {
+      window.dispatchEvent(new Event("agent-panel:open"));
+    }
     transition("idle");
   }, [cleanupTransport, transition]);
 
@@ -538,6 +751,7 @@ function useRealtimeVoiceModeController(
     active: state !== "idle",
     errorMessage: error,
     chatVisible,
+    audioLevels,
     start,
     end,
     toggleChat,
@@ -568,10 +782,10 @@ export function RealtimeVoiceModeProvider({
               state={voice.state === "idle" ? "ending" : voice.state}
               copy={copy}
               chatVisible={voice.chatVisible}
+              audioLevels={voice.audioLevels}
               onToggleChat={voice.toggleChat}
               onEndVoiceMode={voice.end}
               errorMessage={voice.errorMessage}
-              className="z-[270]"
             />,
             document.body,
           )

package/corpus/core/src/client/use-chat-threads.ts

@@ -939,15 +939,39 @@ export function useChatThreads(
           titleSource,
           { preserveUserTitle },
         );
-        await fetch(`${apiUrl}/threads/${encodeURIComponent(id)}`, {
-          method: "PUT",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({
-            ...threadDataPayload,
-            title,
-            scope: localScope,
-          }),
-        });
+        const payload = {
+          ...threadDataPayload,
+          title,
+          scope: localScope,
+        };
+        let response = await fetch(
+          `${apiUrl}/threads/${encodeURIComponent(id)}`,
+          {
+            method: "PUT",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(payload),
+          },
+        );
+        // A passive realtime-voice transcript can be the first content in a
+        // client-created thread, so no agent run has created its SQL row yet.
+        // Materialize that row idempotently and retry the same full save.
+        if (response.status === 404) {
+          const created = await fetch(`${apiUrl}/threads`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ id, title, scope: localScope }),
+          });
+          if (!created.ok) return;
+          response = await fetch(
+            `${apiUrl}/threads/${encodeURIComponent(id)}`,
+            {
+              method: "PUT",
+              headers: { "Content-Type": "application/json" },
+              body: JSON.stringify(payload),
+            },
+          );
+        }
+        if (!response.ok) return;
         emitThreadsUpdated();
         // Update local thread list metadata. If the thread isn't in our
         // local list yet (an optimistic-only thread that the server just

package/corpus/core/src/integrations/a2a-continuation-processor.ts

@@ -18,7 +18,11 @@ import {
   type A2AContinuation,
 } from "./a2a-continuations-store.js";
 import { signInternalToken } from "./internal-token.js";
-import type { PlatformAdapter } from "./types.js";
+import type {
+  OutgoingMessage,
+  PlatformAdapter,
+  PlatformRunProgress,
+} from "./types.js";
 
 const PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-a2a-continuation`;
 const TERMINAL_STATES = new Set(["completed", "failed", "canceled"]);
@@ -150,6 +154,8 @@ async function processClaimedContinuation(
     return;
   }
 
+  const progress = await resumeA2AContinuationProgress(continuation, adapter);
+
   const auth = await signContinuationToken(continuation);
   const client = new A2AClient(continuation.agentUrl, auth.apiKey, {
     requestTimeoutMs: POLL_REQUEST_TIMEOUT_MS,
@@ -162,6 +168,7 @@ async function processClaimedContinuation(
     while (Date.now() < deadline) {
       task = await client.getTask(continuation.a2aTaskId);
       if (TERMINAL_STATES.has(task.status.state)) break;
+      await reportA2AContinuationProgress(continuation, progress, task);
       await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
     }
   } catch (err) {
@@ -171,6 +178,7 @@ async function processClaimedContinuation(
           continuation,
           adapter,
           remotePollFailureReason(continuation),
+          progress,
         );
         return;
       }
@@ -182,6 +190,7 @@ async function processClaimedContinuation(
         continuation,
         adapter,
         err instanceof Error ? err.message : String(err),
+        progress,
       );
       return;
     }
@@ -190,24 +199,12 @@ async function processClaimedContinuation(
   }
 
   if (!task || !TERMINAL_STATES.has(task.status.state)) {
-    const recoverableArtifactText = extractRecoverableArtifactText(task);
-    if (recoverableArtifactText) {
-      await deliverAndCompleteA2AContinuation(
-        continuation,
-        adapter,
-        formatContinuationArtifactText(
-          recoverableArtifactText,
-          continuation.agentUrl,
-        ),
-      );
-      return;
-    }
-
     if (shouldStopPollingRemoteTask(continuation)) {
       await notifyAndFailA2AContinuation(
         continuation,
         adapter,
         remotePollFailureReason(continuation),
+        progress,
       );
       return;
     }
@@ -219,7 +216,7 @@ async function processClaimedContinuation(
     const reason =
       extractTaskText(task) ||
       `Remote A2A task ${continuation.a2aTaskId} ended with state ${task.status.state}`;
-    await notifyAndFailA2AContinuation(continuation, adapter, reason);
+    await notifyAndFailA2AContinuation(continuation, adapter, reason, progress);
     return;
   }
 
@@ -232,11 +229,64 @@ async function processClaimedContinuation(
       continuation,
       adapter,
       `Remote A2A task ${continuation.a2aTaskId} completed without text`,
+      progress,
     );
     return;
   }
 
-  await deliverAndCompleteA2AContinuation(continuation, adapter, text);
+  await deliverAndCompleteA2AContinuation(
+    continuation,
+    adapter,
+    text,
+    progress,
+  );
+}
+
+async function resumeA2AContinuationProgress(
+  continuation: A2AContinuation,
+  adapter: PlatformAdapter,
+): Promise<PlatformRunProgress | null> {
+  if (!continuation.progressRef || !adapter.resumeRunProgress) return null;
+  try {
+    const progress = await adapter.resumeRunProgress(
+      continuation.incoming,
+      continuation.progressRef,
+    );
+    if (!progress) return null;
+    await progress.onEvent({
+      type: "agent_call_progress",
+      agent: continuation.agentName,
+      state: "working",
+      elapsedSeconds: Math.max(
+        0,
+        Math.round((Date.now() - continuation.createdAt) / 1_000),
+      ),
+      detail: "Continuing in the background",
+    });
+    return progress;
+  } catch {
+    // A continuation still has a normal reply fallback. Do not log the
+    // opaque provider reference or the inbound message payload.
+    return null;
+  }
+}
+
+async function reportA2AContinuationProgress(
+  continuation: A2AContinuation,
+  progress: PlatformRunProgress | null,
+  task: Task,
+): Promise<void> {
+  if (!progress) return;
+  await progress.onEvent({
+    type: "agent_call_progress",
+    agent: continuation.agentName,
+    state: task.status.state,
+    elapsedSeconds: Math.max(
+      0,
+      Math.round((Date.now() - continuation.createdAt) / 1_000),
+    ),
+    detail: "Still working on the delegated request",
+  });
 }
 
 async function waitForContinuationDue(
@@ -262,6 +312,7 @@ async function notifyAndFailA2AContinuation(
   continuation: A2AContinuation,
   adapter: PlatformAdapter,
   reason: string,
+  progress: PlatformRunProgress | null = null,
 ): Promise<void> {
   const deliveryContinuation = await claimA2AContinuationDelivery(
     continuation.id,
@@ -273,11 +324,14 @@ async function notifyAndFailA2AContinuation(
     reason,
   );
   try {
+    const outgoing = adapter.formatAgentResponse(message);
     await withTimeout(
-      adapter.sendResponse(
-        adapter.formatAgentResponse(message),
-        deliveryContinuation.incoming,
-        { placeholderRef: deliveryContinuation.placeholderRef ?? undefined },
+      deliverA2AContinuationResponse(
+        adapter,
+        deliveryContinuation,
+        outgoing,
+        progress,
+        "error",
       ),
       PLATFORM_SEND_TIMEOUT_MS,
       `${deliveryContinuation.platform} failure notification timed out`,
@@ -296,6 +350,7 @@ async function deliverAndCompleteA2AContinuation(
   continuation: A2AContinuation,
   adapter: PlatformAdapter,
   text: string,
+  progress: PlatformRunProgress | null = null,
 ): Promise<void> {
   const deliveryContinuation = await claimA2AContinuationDelivery(
     continuation.id,
@@ -303,11 +358,14 @@ async function deliverAndCompleteA2AContinuation(
   if (!deliveryContinuation) return;
 
   try {
+    const outgoing = adapter.formatAgentResponse(text);
     await withTimeout(
-      adapter.sendResponse(
-        adapter.formatAgentResponse(text),
-        deliveryContinuation.incoming,
-        { placeholderRef: deliveryContinuation.placeholderRef ?? undefined },
+      deliverA2AContinuationResponse(
+        adapter,
+        deliveryContinuation,
+        outgoing,
+        progress,
+        "done",
       ),
       PLATFORM_SEND_TIMEOUT_MS,
       `${deliveryContinuation.platform} response delivery timed out`,
@@ -327,6 +385,42 @@ async function deliverAndCompleteA2AContinuation(
   await completeAfterSuccessfulDelivery(deliveryContinuation);
 }
 
+async function deliverA2AContinuationResponse(
+  adapter: PlatformAdapter,
+  continuation: A2AContinuation,
+  message: OutgoingMessage,
+  progress: PlatformRunProgress | null,
+  status: "done" | "error",
+): Promise<void> {
+  if (progress) {
+    try {
+      await progress.onEvent({
+        type: "agent_call",
+        agent: continuation.agentName,
+        status,
+      });
+      await progress.complete(message);
+      return;
+    } catch {
+      // A resumed Slack stream can no longer be finalized (for example when
+      // chat.stopStream rejects). Preserve the final answer with the same
+      // thread reply fallback used by the initial webhook run. Also ask the
+      // adapter to terminate the native stream: otherwise Slack can keep the
+      // task card in its working state after the thread fallback succeeds.
+      try {
+        await progress.fail?.(
+          "I couldn't update the live response, but I posted the final result in this thread.",
+        );
+      } catch {
+        // The thread reply below is still the authoritative final answer.
+      }
+    }
+  }
+  await adapter.sendResponse(message, continuation.incoming, {
+    placeholderRef: continuation.placeholderRef ?? undefined,
+  });
+}
+
 async function rescheduleAndRedispatchA2AContinuation(
   continuationId: string,
 ): Promise<void> {
@@ -524,13 +618,6 @@ function extractTaskText(task: Task): string {
     .join("\n");
 }
 
-function extractRecoverableArtifactText(task: Task | null): string {
-  if (!task?.status.message?.metadata?.agentNativeRecoverableArtifacts) {
-    return "";
-  }
-  return extractTaskText(task);
-}
-
 function formatContinuationArtifactText(
   text: string,
   agentUrl: string,