@agent-native/toolkit 0.221.0-same-host-mcp-20261027-20261027081850 → 0.221.0

package/dist/composer/useRealtimeVoiceMode.js

@@ -1,7 +1,9 @@
 import { jsx as _jsx } from "react/jsx-runtime";
+import { TranscriptGrouper } from "openai/helpers/live";
 import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState, } from "react";
 import { createRealtimeVoiceAudioLevelStore, normalizeRealtimeVoiceRms, smoothRealtimeVoiceLevel, } from "./realtime-voice-audio-level.js";
-import { realtimeVoiceTranscriptRegistry } from "./realtime-voice-transcript.js";
+import { realtimeVoiceTranscriptRegistry, } from "./realtime-voice-transcript.js";
+import { createRealtimeVoiceTurnMetrics } from "./realtime-voice-turn-metrics.js";
 import { useComposerRuntimeAdapters, } from "./runtime-adapters.js";
 import { isMicrophoneAccessError, voiceDictationStartErrorMessage, } from "./useVoiceDictation.js";
 const REALTIME_VOICE_STATE_KEY = "realtime-voice-session";
@@ -11,19 +13,23 @@ const REALTIME_VOICE_TOOL_PATH = "/_agent-native/realtime-voice/tool";
 const REALTIME_VOICE_CAPABILITY_HEADER = "X-Agent-Native-Realtime-Capability";
 const REALTIME_VOICE_PROTOCOL_HEADER = "X-Agent-Native-Realtime-Protocol";
 const REALTIME_VOICE_MODEL_HEADER = "X-Agent-Native-Realtime-Model";
+const REALTIME_VOICE_INTELLIGENCE_HEADER = "X-Agent-Native-Realtime-Intelligence";
+const REALTIME_VOICE_PRIORITY_TOOLS_HEADER = "X-Agent-Native-Realtime-Priority-Tools";
 const REALTIME_VOICE_CONNECTION_TIMEOUT_MS = 15_000;
 const REALTIME_VOICE_MAX_TOOLS = 32;
 const REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES = 32_000;
 const REALTIME_VOICE_MAX_SESSION_BYTES = 64_000;
 const REALTIME_VOICE_TOOL_UPDATE_TIMEOUT_MS = 5_000;
 const REALTIME_VOICE_LIVE_CLOSE_DRAIN_TIMEOUT_MS = 2_000;
-const REALTIME_VOICE_LIVE_TRANSCRIPT_GAP_MS = 750;
 const REALTIME_VOICE_PRIORITY_TOOL_NAMES = [
     "navigate",
     "set-url-path",
     "set-search-params",
     "view-screen",
+    "chat-history",
     "tool-search",
+    "web-search",
+    "web-request",
 ];
 const REALTIME_VOICE_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/;
 const REALTIME_VOICE_MICROPHONE_STORAGE_KEY = "agent-native:realtime-voice-microphone";
@@ -60,6 +66,7 @@ export const DEFAULT_REALTIME_VOICE_PREFERENCES = {
     language: "auto",
     intelligence: "instant",
     voice: "marin",
+    intelligenceChosen: false,
 };
 export const REALTIME_VOICE_AUDIO_CONSTRAINTS = {
     deviceId: { ideal: "default" },
@@ -163,16 +170,31 @@ export function normalizeRealtimeVoicePreferences(value) {
     const record = value && typeof value === "object" && !Array.isArray(value)
         ? value
         : {};
+    const storedIntelligence = isOneOf(REALTIME_VOICE_INTELLIGENCE_LEVELS, record.intelligence)
+        ? record.intelligence
+        : undefined;
+    // Saves from before `intelligenceChosen` stored "instant" for everyone, so
+    // only a non-default level among them shows a pick.
+    const chosenIntelligence = typeof record.intelligenceChosen === "boolean"
+        ? record.intelligenceChosen
+            ? storedIntelligence
+            : undefined
+        : storedIntelligence !== DEFAULT_REALTIME_VOICE_PREFERENCES.intelligence
+            ? storedIntelligence
+            : undefined;
+    // A level saved as not chosen is the app default a session last reported.
+    const appDefaultIntelligence = record.intelligenceChosen === false ? storedIntelligence : undefined;
     return {
         language: isOneOf(REALTIME_VOICE_LANGUAGES, record.language)
             ? record.language
             : DEFAULT_REALTIME_VOICE_PREFERENCES.language,
-        intelligence: isOneOf(REALTIME_VOICE_INTELLIGENCE_LEVELS, record.intelligence)
-            ? record.intelligence
-            : DEFAULT_REALTIME_VOICE_PREFERENCES.intelligence,
+        intelligence: chosenIntelligence ??
+            appDefaultIntelligence ??
+            DEFAULT_REALTIME_VOICE_PREFERENCES.intelligence,
         voice: isOneOf(REALTIME_VOICES, record.voice)
             ? record.voice
             : DEFAULT_REALTIME_VOICE_PREFERENCES.voice,
+        intelligenceChosen: chosenIntelligence !== undefined,
     };
 }
 export function resolveRealtimeVoiceLanguage(language, browserLanguages = []) {
@@ -187,11 +209,19 @@ export function resolveRealtimeVoiceLanguage(language, browserLanguages = []) {
     return "en";
 }
 export function realtimeVoiceReasoningEffort(intelligence, protocol) {
+    // The delegated Live model rejects "minimal", so its levels start at "low".
+    if (protocol === "live") {
+        if (intelligence === "balanced")
+            return "medium";
+        if (intelligence === "deep")
+            return "high";
+        return "low";
+    }
     if (intelligence === "balanced")
         return "low";
     if (intelligence === "deep")
         return "medium";
-    return protocol === "live" ? "low" : "minimal";
+    return "minimal";
 }
 export function createRealtimeVoicePreferenceUpdate(preferences, options = {}) {
     if (options.protocol === "live") {
@@ -237,32 +267,131 @@ export function createRealtimeVoiceLanguageUpdate(language, browserLanguages = [
         content: `Speak in ${resolvedLanguage} unless the user asks to switch languages.`,
     };
 }
+function publishRealtimeVoiceWorkSegment(segment, completedAt) {
+    if (segment.calls.length === 0)
+        return;
+    realtimeVoiceTranscriptRegistry.publishWork({
+        id: segment.id,
+        threadId: segment.threadId,
+        calls: segment.calls.map((call) => ({ ...call })),
+        ...(completedAt ? { completedAt } : {}),
+    });
+}
+/**
+ * The reply's open work a new call joins, or new work; none without a thread.
+ * A call from a different delegation is a new request.
+ */
+function openRealtimeVoiceWorkSegment(ref, threadId, callId, delegationId) {
+    if (!threadId)
+        return null;
+    const current = ref.current;
+    if (current?.threadId === threadId &&
+        (!current.delegationId ||
+            !delegationId ||
+            current.delegationId === delegationId)) {
+        current.delegationId ??= delegationId;
+        return current;
+    }
+    if (current)
+        closeRealtimeVoiceWorkSegment(ref, current);
+    const segment = {
+        id: callId,
+        threadId,
+        delegationId,
+        calls: [],
+        closing: false,
+    };
+    ref.current = segment;
+    return segment;
+}
+/** Ends a call that waited for approval: `result` is null when it never ran. */
+function finishPendingApproval(pending, result) {
+    const { workCall, workSegment } = pending;
+    workCall.status =
+        !result || result.status === "approval_required"
+            ? "cancelled"
+            : result.status;
+    if (result)
+        workCall.output = result.output;
+    workCall.completedAt = new Date().toISOString();
+    if (!workSegment)
+        return;
+    if (workSegment.closing) {
+        closeRealtimeVoiceWorkSegment({ current: null }, workSegment);
+    }
+    else {
+        publishRealtimeVoiceWorkSegment(workSegment);
+    }
+}
+function closeRealtimeVoiceWorkSegment(ref, segment = ref.current) {
+    if (!segment)
+        return;
+    if (ref.current === segment)
+        ref.current = null;
+    segment.closing = true;
+    if (segment.calls.some((call) => call.status === "running"))
+        return;
+    const completedAt = segment.calls
+        .map((call) => call.completedAt ?? call.startedAt)
+        .sort()
+        .at(-1);
+    publishRealtimeVoiceWorkSegment(segment, completedAt);
+}
 /**
  * Orders provider transcripts into conversation order. `publish` receives each
  * final transcript; `stream`, when given, receives the growing text of a turn
- * under the same `providerId` its final transcript will carry.
+ * under the same `providerId` its final transcript will carry. GPT-Live turns
+ * come from the SDK's transcript grouper, which splits on speaker changes and
+ * two-second silences and drops short backchannels heard over the reply.
  */
-export function createRealtimeVoiceTranscriptSequencer(publish, stream) {
+export function createRealtimeVoiceTranscriptSequencer(publish, stream, 
+/** Each GPT-Live turn as it grows, with its place on the session clock. */
+onLiveTurn) {
     const order = [];
     const items = new Map();
     const settledItemIds = new Set();
     const receivedTranscriptIds = new Set();
-    const liveBuffers = new Map();
     const streamingText = new Map();
     const streamedText = new Map();
-    let liveSequence = 0;
-    const liveProviderId = (buffer) => `live-${buffer.sequence}`;
-    const flushLiveTranscript = (role) => {
-        const buffer = liveBuffers.get(role);
-        if (!buffer)
-            return;
-        liveBuffers.delete(role);
-        const text = buffer.text.trim();
+    let liveGrouper;
+    const liveTranscript = (segment) => {
+        const text = segment.text.trim();
         if (!text)
+            return undefined;
+        return stream
+            ? { role: segment.speaker, text, providerId: segment.id }
+            : { role: segment.speaker, text };
+    };
+    // Listeners check that their grouper is still current, so a discarded one
+    // never publishes what it closes on the way out.
+    const openLiveGrouper = () => {
+        if (liveGrouper)
+            return liveGrouper;
+        const grouper = new TranscriptGrouper();
+        grouper.on("segment.updated", (segment) => {
+            if (grouper !== liveGrouper)
+                return;
+            onLiveTurn?.(segment);
+            const transcript = liveTranscript(segment);
+            if (stream && transcript)
+                stream(transcript);
+        });
+        grouper.on("segment.closed", ({ segment }) => {
+            const transcript = liveTranscript(segment);
+            if (grouper === liveGrouper && transcript)
+                publish(transcript);
+        });
+        liveGrouper = grouper;
+        return grouper;
+    };
+    const closeLiveGrouper = (keepText) => {
+        const grouper = liveGrouper;
+        if (!grouper)
             return;
-        publish(stream
-            ? { role, text, providerId: liveProviderId(buffer) }
-            : { role, text });
+        if (!keepText)
+            liveGrouper = undefined;
+        grouper.close();
+        liveGrouper = undefined;
     };
     // A turn appears once every earlier turn has appeared, so a fast reply
     // never streams in above the question it answers.
@@ -289,11 +418,6 @@ export function createRealtimeVoiceTranscriptSequencer(publish, stream) {
             });
         }
     };
-    const flushLiveTranscripts = () => {
-        for (const buffer of [...liveBuffers.values()].sort((left, right) => left.sequence - right.sequence)) {
-            flushLiveTranscript(buffer.role);
-        }
-    };
     const reserve = (id, role) => {
         if (settledItemIds.has(id))
             return;
@@ -333,35 +457,17 @@ export function createRealtimeVoiceTranscriptSequencer(publish, stream) {
         handle(event) {
             if (event.type === "session.input_transcript.delta" ||
                 event.type === "session.output_transcript.delta") {
-                const role = event.type === "session.input_transcript.delta"
-                    ? "user"
-                    : "assistant";
-                const startMs = typeof event.start_ms === "number" ? event.start_ms : undefined;
-                const endMs = typeof event.end_ms === "number" ? event.end_ms : startMs;
-                let buffer = liveBuffers.get(role);
-                if (buffer &&
-                    startMs !== undefined &&
-                    buffer.lastEndMs !== undefined &&
-                    startMs - buffer.lastEndMs > REALTIME_VOICE_LIVE_TRANSCRIPT_GAP_MS) {
-                    flushLiveTranscript(role);
-                    buffer = undefined;
-                }
-                if (!buffer) {
-                    buffer = {
-                        role,
-                        text: "",
-                        sequence: ++liveSequence,
-                    };
-                    liveBuffers.set(role, buffer);
-                }
-                if (typeof event.delta === "string")
-                    buffer.text += event.delta;
-                if (endMs !== undefined)
-                    buffer.lastEndMs = endMs;
-                const text = buffer.text.trim();
-                if (stream && text) {
-                    stream({ role, text, providerId: liveProviderId(buffer) });
+                const type = event.type === "session.input_transcript.delta"
+                    ? "session.input_transcript.delta"
+                    : "session.output_transcript.delta";
+                const { event_id, delta, start_ms, end_ms } = event;
+                if (typeof event_id !== "string" ||
+                    typeof delta !== "string" ||
+                    typeof start_ms !== "number" ||
+                    typeof end_ms !== "number") {
+                    throw new Error("GPT-Live sent a transcript delta without an id, text, or timing.");
                 }
+                openLiveGrouper().push({ type, event_id, delta, start_ms, end_ms });
                 return;
             }
             if (event.type === "conversation.item.input_audio_transcription.delta" ||
@@ -379,7 +485,7 @@ export function createRealtimeVoiceTranscriptSequencer(publish, stream) {
                 return;
             }
             if (event.type === "session.closed")
-                flushLiveTranscripts();
+                closeLiveGrouper(true);
             if (event.type === "conversation.item.added" ||
                 event.type === "conversation.item.created") {
                 const item = event.item;
@@ -455,7 +561,7 @@ export function createRealtimeVoiceTranscriptSequencer(publish, stream) {
             emitStreams();
         },
         flush() {
-            flushLiveTranscripts();
+            closeLiveGrouper(true);
             if (!stream)
                 return;
             // The session is over: a turn that streamed but never got its final
@@ -482,10 +588,11 @@ export function createRealtimeVoiceTranscriptSequencer(publish, stream) {
             }
         },
         reset(options) {
-            liveBuffers.clear();
-            liveSequence = 0;
+            // A failed response leaves what was already heard; a new session
+            // starts clean.
             if (options?.preserveQueuedTranscripts)
                 return;
+            closeLiveGrouper(false);
             order.length = 0;
             items.clear();
             settledItemIds.clear();
@@ -688,7 +795,11 @@ export async function createRealtimeVoiceSessionWithCapability(offerSdp, options
             ...(preferences
                 ? {
                     "X-Agent-Native-Realtime-Language": resolveRealtimeVoiceLanguage(preferences.language, options.browserLanguages),
-                    "X-Agent-Native-Realtime-Intelligence": preferences.intelligence,
+                    ...(preferences.intelligenceChosen !== false
+                        ? {
+                            [REALTIME_VOICE_INTELLIGENCE_HEADER]: preferences.intelligence,
+                        }
+                        : {}),
                     "X-Agent-Native-Realtime-Voice": preferences.voice,
                 }
                 : {}),
@@ -710,11 +821,24 @@ export async function createRealtimeVoiceSessionWithCapability(offerSdp, options
         : undefined;
     const modelHeader = response.headers.get(REALTIME_VOICE_MODEL_HEADER);
     const model = modelHeader?.trim() || undefined;
+    const intelligenceHeader = response.headers
+        .get(REALTIME_VOICE_INTELLIGENCE_HEADER)
+        ?.trim();
+    const intelligence = isOneOf(REALTIME_VOICE_INTELLIGENCE_LEVELS, intelligenceHeader)
+        ? intelligenceHeader
+        : undefined;
+    const priorityToolNames = (response.headers.get(REALTIME_VOICE_PRIORITY_TOOLS_HEADER) ?? "")
+        .split(",")
+        .map((name) => name.trim())
+        .filter((name) => REALTIME_VOICE_TOOL_NAME.test(name))
+        .slice(0, REALTIME_VOICE_MAX_TOOLS);
     return {
         sdp,
         ...(capability ? { capability } : {}),
         ...(protocol ? { protocol } : {}),
         ...(model ? { model } : {}),
+        ...(intelligence ? { intelligence } : {}),
+        ...(priorityToolNames.length > 0 ? { priorityToolNames } : {}),
     };
 }
 export async function createRealtimeVoiceSession(offerSdp, options = {}) {
@@ -742,6 +866,10 @@ export async function executeRealtimeVoiceTool(input) {
             callId: input.callId,
             sessionId: input.sessionId,
             browserTabId: input.browserTabId,
+            ...(input.approvalId ? { approvalId: input.approvalId } : {}),
+            ...(input.declinedApprovalId
+                ? { declinedApprovalId: input.declinedApprovalId }
+                : {}),
         }),
         signal: input.signal,
     });
@@ -814,7 +942,11 @@ function createRealtimeVoiceToolManifestUpdate(tools, eventId, protocol = "realt
             ? {
                 delegation: {
                     type: "responses",
-                    responses: { tools, tool_choice: "auto" },
+                    // Strict schemas make every optional parameter required.
+                    responses: {
+                        tools: tools.map((tool) => ({ ...tool, strict: false })),
+                        tool_choice: "auto",
+                    },
                 },
             }
             : {
@@ -824,7 +956,7 @@ function createRealtimeVoiceToolManifestUpdate(tools, eventId, protocol = "realt
             },
     };
 }
-export function mergeRealtimeVoiceToolManifest(currentTools, expandedTools, protocol = "realtime") {
+export function mergeRealtimeVoiceToolManifest(currentTools, expandedTools, protocol = "realtime", appPriorityToolNames = []) {
     const current = new Map();
     for (const value of currentTools) {
         const tool = normalizeRealtimeVoiceFunctionTool(value);
@@ -845,7 +977,10 @@ export function mergeRealtimeVoiceToolManifest(currentTools, expandedTools, prot
         seen.add(tool.name);
         candidates.push(tool);
     };
-    for (const name of REALTIME_VOICE_PRIORITY_TOOL_NAMES) {
+    for (const name of [
+        ...REALTIME_VOICE_PRIORITY_TOOL_NAMES,
+        ...appPriorityToolNames,
+    ]) {
         add(expanded.get(name) ?? current.get(name));
     }
     for (const tool of expanded.values())
@@ -863,6 +998,25 @@ export function mergeRealtimeVoiceToolManifest(currentTools, expandedTools, prot
     }
     return packed;
 }
+/** Tells the assistant what the user decided about a call that asked. */
+export function createRealtimeVoiceApprovalOutcomeEvent(toolName, result, protocol = "realtime") {
+    const text = !result
+        ? `The user declined ${toolName}, so it did not run.`
+        : result.status === "approval_required"
+            ? `${toolName} asked for approval again, so it has not run. The user can approve it on screen.`
+            : `The user approved ${toolName}. ${JSON.stringify({
+                status: result.status,
+                output: result.output,
+            })}`;
+    return {
+        type: protocol === "live" ? "response.item.create" : "conversation.item.create",
+        item: {
+            type: "message",
+            role: "system",
+            content: [{ type: "input_text", text }],
+        },
+    };
+}
 function createRealtimeVoiceToolResultEvent(result, protocol = "realtime") {
     const modelResult = {
         callId: result.callId,
@@ -882,6 +1036,7 @@ function createRealtimeVoiceToolResultEvent(result, protocol = "realtime") {
 export function createRealtimeVoiceToolManifestCoordinator(send, timeoutMs = REALTIME_VOICE_TOOL_UPDATE_TIMEOUT_MS) {
     let currentTools = [];
     let protocol = "realtime";
+    let priorityToolNames = [];
     const queue = [];
     let updateSequence = 0;
     let active;
@@ -918,7 +1073,7 @@ export function createRealtimeVoiceToolManifestCoordinator(send, timeoutMs = REA
             drain();
             return;
         }
-        const merged = mergeRealtimeVoiceToolManifest(currentTools, expanded, protocol);
+        const merged = mergeRealtimeVoiceToolManifest(currentTools, expanded, protocol, priorityToolNames);
         const mergedNames = new Set(merged.map((tool) => tool.name));
         const expectedNames = expanded
             .map((tool) => tool.name)
@@ -946,8 +1101,11 @@ export function createRealtimeVoiceToolManifestCoordinator(send, timeoutMs = REA
         setProtocol(nextProtocol) {
             protocol = nextProtocol;
         },
+        setPriorityToolNames(names) {
+            priorityToolNames = [...names];
+        },
         setSessionTools(tools) {
-            currentTools = mergeRealtimeVoiceToolManifest([], tools, protocol);
+            currentTools = mergeRealtimeVoiceToolManifest([], tools, protocol, priorityToolNames);
             if (!active)
                 return;
             const names = new Set(currentTools.map((tool) => tool.name));
@@ -965,8 +1123,10 @@ export function createRealtimeVoiceToolManifestCoordinator(send, timeoutMs = REA
                 clearTimeout(active.timer);
             active = undefined;
             queue.length = 0;
-            if (!options?.preserveSessionTools)
+            if (!options?.preserveSessionTools) {
                 currentTools = [];
+                priorityToolNames = [];
+            }
             protocol = "realtime";
         },
         getTools() {
@@ -1099,6 +1259,15 @@ export function createRealtimeVoiceModeCopy(t) {
         voiceSettings: t("agentChat.voiceMode.voiceSettings", {
             defaultValue: "Voice settings",
         }),
+        thinking: t("agentChat.status.thinking", { defaultValue: "Thinking" }),
+        approval: {
+            question: (toolLabel) => t("agentChat.approval.question", {
+                tool: toolLabel,
+                defaultValue: "Approve to run {{tool}}?",
+            }),
+            approve: t("agentChat.approval.approve", { defaultValue: "Approve" }),
+            deny: t("agentChat.approval.deny", { defaultValue: "Deny" }),
+        },
         settings: {
             language: t("agentChat.voiceMode.settings.language", {
                 defaultValue: "Language",
@@ -1289,16 +1458,43 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
     const lastUserTextRef = useRef("");
     const lastAssistantTextRef = useRef("");
     const transcriptThreadIdRef = useRef(undefined);
+    const requestedTranscriptThreadIdRef = useRef(undefined);
     const liveCloseDrainTimerRef = useRef(null);
     const liveCloseDrainFinishRef = useRef(null);
     const toolAbortControllersRef = useRef(new Set());
+    const pendingApprovalsRef = useRef(new Map());
+    const runningToolsRef = useRef(new Map());
+    const [currentToolName, setCurrentToolName] = useState(null);
+    const trackRunningTool = useCallback((callId, name) => {
+        const running = runningToolsRef.current;
+        if (name)
+            running.set(callId, name);
+        else
+            running.delete(callId);
+        setCurrentToolName([...running.values()].at(-1) ?? null);
+    }, []);
+    const [approvals, setApprovals] = useState([]);
+    const syncApprovals = useCallback(() => {
+        setApprovals([...pendingApprovalsRef.current.values()].map(({ callId, name }) => ({
+            callId,
+            toolName: name,
+        })));
+    }, []);
     const transcriptSequenceRef = useRef(0);
+    const transcriptCreatedAtRef = useRef(new Map());
+    const workSegmentRef = useRef(null);
+    const responseCallCountsRef = useRef(new Map());
     const preferencesHydratedRef = useRef(false);
     const preferencesEditedRef = useRef(false);
     const preferenceWriteChainRef = useRef(Promise.resolve());
     const responseCoordinator = useMemo(() => createRealtimeVoiceResponseCoordinator((event) => {
         sendDataChannelEvent(channelRef.current, event);
     }), []);
+    // The metrics outlive adapter changes such as a locale switch: the
+    // transport cleanup depends on them, and a new instance would end the call.
+    const trackEventRef = useRef(adapters.voice.trackEvent);
+    trackEventRef.current = adapters.voice.trackEvent;
+    const turnMetrics = useMemo(() => createRealtimeVoiceTurnMetrics((name, properties) => trackEventRef.current(name, properties)), []);
     const toolManifestCoordinator = useMemo(() => createRealtimeVoiceToolManifestCoordinator((event) => {
         if (event.type === "response.create") {
             responseCoordinator.request(event);
@@ -1373,12 +1569,17 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
             .catch(() => undefined);
     }, [adapters]);
     const updateLivePreferences = useCallback((next, includeVoice = false) => {
+        turnMetrics.setContext({
+            protocol: protocolRef.current,
+            intelligence: next.intelligence,
+            intelligenceSource: next.intelligenceChosen ? "user" : "default",
+        });
         sendDataChannelEvent(channelRef.current, createRealtimeVoicePreferenceUpdate(next, {
             browserLanguages: typeof navigator === "undefined" ? [] : navigator.languages,
             includeVoice,
             protocol: protocolRef.current,
         }));
-    }, []);
+    }, [turnMetrics]);
     const setLanguage = useCallback((language) => {
         const next = { ...preferencesRef.current, language };
         savePreferences(next);
@@ -1388,7 +1589,11 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         updateLivePreferences(next);
     }, [savePreferences, updateLivePreferences]);
     const setIntelligence = useCallback((intelligence) => {
-        const next = { ...preferencesRef.current, intelligence };
+        const next = {
+            ...preferencesRef.current,
+            intelligence,
+            intelligenceChosen: true,
+        };
         savePreferences(next);
         updateLivePreferences(next);
     }, [savePreferences, updateLivePreferences]);
@@ -1519,6 +1724,19 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
     ]);
     switchMicrophoneRef.current = setMicrophone;
     const cleanupTransport = useCallback(() => {
+        // Teardown keeps what was heard now, so no grouper timer publishes into
+        // the chat after the session is gone.
+        flushTranscriptsRef.current?.();
+        turnMetrics.sessionEnded();
+        for (const pending of pendingApprovalsRef.current.values()) {
+            finishPendingApproval(pending, null);
+        }
+        pendingApprovalsRef.current.clear();
+        syncApprovals();
+        runningToolsRef.current.clear();
+        setCurrentToolName(null);
+        closeRealtimeVoiceWorkSegment(workSegmentRef);
+        responseCallCountsRef.current.clear();
         if (liveCloseDrainTimerRef.current !== null) {
             window.clearTimeout(liveCloseDrainTimerRef.current);
             liveCloseDrainTimerRef.current = null;
@@ -1570,7 +1788,9 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         abortActiveToolCalls,
         audioLevels,
         responseCoordinator,
+        syncApprovals,
         toolManifestCoordinator,
+        turnMetrics,
     ]);
     const fail = useCallback((message, options) => {
         flushTranscriptsRef.current?.();
@@ -1580,7 +1800,46 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         if (options?.openKeySettings)
             openOpenAiKeySettings();
     }, [cleanupTransport, transition]);
-    const handleFunctionCall = useCallback(async (call) => {
+    const publishTranscript = useCallback((transcript, status) => {
+        const threadId = transcriptThreadIdRef.current;
+        if (!threadId)
+            return;
+        // The session id arrives after the first events; the start time keeps
+        // every update of one turn under the same message id.
+        const sessionIdentity = startedAtRef.current ?? "pending";
+        const providerIdentity = transcript.providerId ?? `sequence-${++transcriptSequenceRef.current}`;
+        const id = `realtime-voice:${sessionIdentity}:${transcript.role}:${providerIdentity}`;
+        // Work rows sit before the first message created after their call, so a
+        // turn keeps the time it started however late its final text arrives.
+        let createdAt = transcriptCreatedAtRef.current.get(id);
+        if (!createdAt) {
+            createdAt = new Date().toISOString();
+            transcriptCreatedAtRef.current.set(id, createdAt);
+        }
+        realtimeVoiceTranscriptRegistry.publish({
+            id,
+            threadId,
+            role: transcript.role,
+            text: transcript.text,
+            createdAt,
+            status,
+        });
+    }, []);
+    const transcriptSequencer = useMemo(() => createRealtimeVoiceTranscriptSequencer((transcript) => publishTranscript(transcript, "complete"), (transcript) => publishTranscript(transcript, "streaming"), (turn) => {
+        if (turn.speaker !== "user")
+            return;
+        turnMetrics.userTurnUpdated(performance.now(), turn.id, turn.endMs);
+    }), [publishTranscript, turnMetrics]);
+    flushTranscriptsRef.current = transcriptSequencer.flush;
+    const greetingStarter = useMemo(() => createRealtimeVoiceGreetingStarter((event) => {
+        if (event.type === "response.create") {
+            responseCoordinator.request(event);
+        }
+        else {
+            sendDataChannelEvent(channelRef.current, event);
+        }
+    }), [responseCoordinator]);
+    const handleFunctionCall = useCallback(async (call, delegationId) => {
         if (handledCallsRef.current.has(call.callId))
             return;
         handledCallsRef.current.add(call.callId);
@@ -1589,9 +1848,23 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         const toolAbortController = new AbortController();
         toolAbortControllersRef.current.add(toolAbortController);
         transition("working");
+        trackRunningTool(call.callId, call.name);
+        const finishTurnToolCall = turnMetrics.toolCallStarted();
+        const workCall = {
+            id: call.callId,
+            name: call.name,
+            status: "running",
+            startedAt: new Date().toISOString(),
+        };
+        const workSegment = openRealtimeVoiceWorkSegment(workSegmentRef, transcriptThreadIdRef.current, call.callId, delegationId);
         let result;
         try {
             const args = parseFunctionArguments(call.argumentsText);
+            workCall.input = args;
+            if (workSegment) {
+                workSegment.calls.push(workCall);
+                publishRealtimeVoiceWorkSegment(workSegment);
+            }
             result = await executeRealtimeVoiceTool({
                 name: call.name,
                 args,
@@ -1611,42 +1884,143 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         }
         finally {
             toolAbortControllersRef.current.delete(toolAbortController);
+            trackRunningTool(call.callId, null);
         }
+        // Session teardown aborted the call, so it did not fail. A call that
+        // asks for approval stays running until the user decides.
+        const approvalId = !toolAbortController.signal.aborted &&
+            result.status === "approval_required" &&
+            workCall.input &&
+            transportGeneration === transportGenerationRef.current
+            ? result.approvalId
+            : undefined;
+        const awaitingApproval = Boolean(approvalId);
+        if (!awaitingApproval) {
+            workCall.status =
+                toolAbortController.signal.aborted ||
+                    result.status === "approval_required"
+                    ? "cancelled"
+                    : result.status;
+            workCall.output = result.output;
+            workCall.completedAt = new Date().toISOString();
+        }
+        if (workSegment) {
+            if (!workSegment.calls.includes(workCall)) {
+                workSegment.calls.push(workCall);
+            }
+            if (workSegment.closing) {
+                closeRealtimeVoiceWorkSegment(workSegmentRef, workSegment);
+            }
+            else {
+                publishRealtimeVoiceWorkSegment(workSegment);
+            }
+        }
+        finishTurnToolCall(performance.now(), toolAbortController.signal.aborted ? { status: "cancelled" } : result);
         if (transportGeneration !== transportGenerationRef.current ||
             channelRef.current !== transportChannel) {
             return;
         }
         if (result.capability)
             capabilityRef.current = result.capability;
+        if (approvalId && workCall.input) {
+            pendingApprovalsRef.current.set(call.callId, {
+                callId: call.callId,
+                name: call.name,
+                args: workCall.input,
+                approvalId,
+                workCall,
+                workSegment,
+            });
+            syncApprovals();
+        }
         toolManifestCoordinator.enqueue(result);
-    }, [browserTabId, toolManifestCoordinator, transition]);
-    const publishTranscript = useCallback((transcript, status) => {
-        const threadId = transcriptThreadIdRef.current;
-        if (!threadId)
+    }, [
+        browserTabId,
+        syncApprovals,
+        toolManifestCoordinator,
+        trackRunningTool,
+        transition,
+        turnMetrics,
+    ]);
+    const resolveApproval = useCallback(async (callId, approved) => {
+        const pending = pendingApprovalsRef.current.get(callId);
+        if (!pending)
             return;
-        // The session id arrives after the first events; the start time keeps
-        // every update of one turn under the same message id.
-        const sessionIdentity = startedAtRef.current ?? "pending";
-        const providerIdentity = transcript.providerId ?? `sequence-${++transcriptSequenceRef.current}`;
-        realtimeVoiceTranscriptRegistry.publish({
-            id: `realtime-voice:${sessionIdentity}:${transcript.role}:${providerIdentity}`,
-            threadId,
-            role: transcript.role,
-            text: transcript.text,
-            createdAt: new Date().toISOString(),
-            status,
-        });
-    }, []);
-    const transcriptSequencer = useMemo(() => createRealtimeVoiceTranscriptSequencer((transcript) => publishTranscript(transcript, "complete"), (transcript) => publishTranscript(transcript, "streaming")), [publishTranscript]);
-    flushTranscriptsRef.current = transcriptSequencer.flush;
-    const greetingStarter = useMemo(() => createRealtimeVoiceGreetingStarter((event) => {
-        if (event.type === "response.create") {
-            responseCoordinator.request(event);
+        pendingApprovalsRef.current.delete(callId);
+        syncApprovals();
+        const transportGeneration = transportGenerationRef.current;
+        const transportChannel = channelRef.current;
+        let result = null;
+        let aborted = false;
+        if (approved) {
+            transition("working");
+            trackRunningTool(callId, pending.name);
+            const toolAbortController = new AbortController();
+            toolAbortControllersRef.current.add(toolAbortController);
+            try {
+                result = await executeRealtimeVoiceTool({
+                    name: pending.name,
+                    args: pending.args,
+                    callId: `${callId}_approved`,
+                    sessionId: sessionIdRef.current,
+                    browserTabId,
+                    capability: capabilityRef.current,
+                    signal: toolAbortController.signal,
+                    approvalId: pending.approvalId,
+                });
+            }
+            catch (toolError) {
+                result = {
+                    callId,
+                    status: "failed",
+                    output: errorMessage(toolError),
+                };
+            }
+            finally {
+                toolAbortControllersRef.current.delete(toolAbortController);
+                trackRunningTool(callId, null);
+            }
+            aborted = toolAbortController.signal.aborted;
         }
         else {
-            sendDataChannelEvent(channelRef.current, event);
+            // A declined record left pending would make a later identical
+            // request's approval ambiguous on the server.
+            await executeRealtimeVoiceTool({
+                name: pending.name,
+                args: pending.args,
+                callId: `${callId}_declined`,
+                sessionId: sessionIdRef.current,
+                browserTabId,
+                capability: capabilityRef.current,
+                declinedApprovalId: pending.approvalId,
+            }).catch((declineError) => {
+                console.warn("Could not record the declined voice approval.", declineError);
+            });
         }
-    }), [responseCoordinator]);
+        const transportChanged = transportGeneration !== transportGenerationRef.current ||
+            channelRef.current !== transportChannel;
+        // An approval that expired or no longer applies asks again; the call
+        // stays running and the user sees the new request.
+        const renewedApprovalId = !aborted && !transportChanged && result?.status === "approval_required"
+            ? result.approvalId
+            : undefined;
+        if (renewedApprovalId) {
+            pendingApprovalsRef.current.set(callId, {
+                ...pending,
+                approvalId: renewedApprovalId,
+            });
+            syncApprovals();
+        }
+        else {
+            finishPendingApproval(pending, aborted || !result ? null : result);
+        }
+        if (aborted || transportChanged)
+            return;
+        if (result?.capability)
+            capabilityRef.current = result.capability;
+        sendDataChannelEvent(channelRef.current, createRealtimeVoiceApprovalOutcomeEvent(pending.name, result, protocolRef.current));
+        sendDataChannelEvent(channelRef.current, { type: "response.create" });
+    }, [browserTabId, syncApprovals, trackRunningTool, transition]);
     const handleServerEvent = useCallback((event, draining = false) => {
         if (draining) {
             transcriptSequencer.handle(event);
@@ -1686,9 +2060,12 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
             transition("working");
         }
         else if (event.type === "input_audio_buffer.speech_started") {
+            closeRealtimeVoiceWorkSegment(workSegmentRef);
+            turnMetrics.userSpeechStarted(performance.now());
             transition("listening");
         }
         else if (event.type === "input_audio_buffer.speech_stopped") {
+            turnMetrics.userSpeechEnded(performance.now());
             transition("working");
             if (protocolRef.current === "realtime")
                 responseCoordinator.request();
@@ -1698,7 +2075,11 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
             lastAssistantTextRef.current = "";
             transition("working");
         }
+        else if (event.type === "output_audio_buffer.started") {
+            turnMetrics.assistantAudioStarted(performance.now());
+        }
         else if (event.type === "response.output_audio_transcript.delta") {
+            turnMetrics.assistantResponded();
             hasOutputAudioRef.current = true;
             if (typeof event.delta === "string") {
                 lastAssistantTextRef.current += event.delta;
@@ -1724,6 +2105,7 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
             transition("listening");
         }
         else if (event.type === "session.output_transcript.delta") {
+            turnMetrics.assistantAudioStarted(performance.now(), typeof event.start_ms === "number" ? event.start_ms : undefined);
             hasOutputAudioRef.current = true;
             if (typeof event.delta === "string") {
                 lastAssistantTextRef.current += event.delta;
@@ -1741,6 +2123,7 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
                 ? response.status
                 : undefined;
             if (responseEvent.type === "response.failed" || status === "failed") {
+                closeRealtimeVoiceWorkSegment(workSegmentRef);
                 transportGenerationRef.current += 1;
                 handledCallsRef.current.clear();
                 abortActiveToolCalls();
@@ -1783,12 +2166,30 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
                 }));
             return;
         }
+        const delegationId = typeof event.delegation_id === "string" ? event.delegation_id : "";
         const calls = extractRealtimeVoiceFunctionCalls(event);
-        for (const call of calls)
-            void handleFunctionCall(call);
-        if (responseBoundary && calls.length === 0) {
+        for (const call of calls) {
+            void handleFunctionCall(call, delegationId || undefined);
+        }
+        // GPT-Live's response snapshots carry no output, so whether a response
+        // called a tool is counted from its streamed items, per delegation.
+        const responseCalls = responseCallCountsRef.current;
+        if (responseEvent.type === "response.created") {
+            responseCalls.set(delegationId, 0);
+        }
+        if (calls.length > 0) {
+            responseCalls.set(delegationId, (responseCalls.get(delegationId) ?? 0) + calls.length);
+        }
+        if (responseBoundary && !responseCalls.get(delegationId)) {
+            // Only the segment's own delegation ends it; an uncorrelated
+            // boundary would split one request's work into several rows.
+            if ((workSegmentRef.current?.delegationId ?? "") === delegationId) {
+                closeRealtimeVoiceWorkSegment(workSegmentRef);
+            }
             transition("listening");
         }
+        if (responseBoundary)
+            responseCalls.delete(delegationId);
         if (responseBoundary) {
             lastUserTextRef.current = "";
             lastAssistantTextRef.current = "";
@@ -1805,6 +2206,7 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         transition,
         responseCoordinator,
         toolManifestCoordinator,
+        turnMetrics,
         updateLivePreferences,
     ]);
     const start = useCallback(async () => {
@@ -1829,8 +2231,10 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         greetingStarter.reset();
         setVoiceChangePending(false);
         transcriptThreadIdRef.current =
-            realtimeVoiceTranscriptRegistry.activeThreadId();
+            requestedTranscriptThreadIdRef.current ??
+                realtimeVoiceTranscriptRegistry.activeThreadId();
         transcriptSequenceRef.current = 0;
+        transcriptCreatedAtRef.current.clear();
         transcriptSequencer.reset();
         inputMutedRef.current = false;
         outputMutedRef.current = false;
@@ -1904,8 +2308,10 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
                 try {
                     handleServerEvent(JSON.parse(String(messageEvent.data)), stateRef.current === "ending");
                 }
-                catch {
-                    // Ignore malformed provider events without ending a healthy call.
+                catch (eventError) {
+                    // A malformed provider event must not end a healthy call, but the
+                    // text it carried is lost, so say so.
+                    console.warn("Ignored a realtime voice event that could not be handled.", eventError);
                 }
             };
             channel.onerror = () => {
@@ -1955,7 +2361,25 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
                 answer.model ??
                     (protocolRef.current === "live" ? "gpt-live-1" : "gpt-realtime-2.1");
             toolManifestCoordinator.setProtocol(protocolRef.current);
+            toolManifestCoordinator.setPriorityToolNames(answer.priorityToolNames ?? []);
             greetingStarter.setProtocol(protocolRef.current);
+            const chosen = preferencesRef.current.intelligenceChosen === true;
+            if (!chosen &&
+                answer.intelligence &&
+                answer.intelligence !== preferencesRef.current.intelligence) {
+                // Saved as not chosen, so Settings shows the app's default after a
+                // reload and the app can still change it.
+                savePreferences({
+                    ...preferencesRef.current,
+                    intelligence: answer.intelligence,
+                    intelligenceChosen: false,
+                });
+            }
+            turnMetrics.setContext({
+                protocol: protocolRef.current,
+                intelligence: preferencesRef.current.intelligence,
+                intelligenceSource: chosen ? "user" : "default",
+            });
             await peer.setRemoteDescription({ type: "answer", sdp: answer.sdp });
         }
         catch (startError) {
@@ -1990,10 +2414,16 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         handleServerEvent,
         hydratePreferences,
         refreshMicrophones,
+        savePreferences,
         t,
         transcriptSequencer,
         transition,
+        turnMetrics,
     ]);
+    const startForThread = useCallback((options) => {
+        requestedTranscriptThreadIdRef.current = options?.threadId;
+        return start();
+    }, [start]);
     const end = useCallback(() => {
         if (stateRef.current === "idle" || stateRef.current === "ending")
             return;
@@ -2107,9 +2537,12 @@ function useRealtimeVoiceModeController(browserTabId, copy) {
         setMicrophone,
         setInputMuted,
         setOutputMuted,
-        start,
+        start: startForThread,
         end,
         dismissError,
+        currentToolName,
+        approvals,
+        resolveApproval,
     };
 }
 const RealtimeVoiceModeContext = createContext(null);
@@ -2170,7 +2603,11 @@ export function buildRealtimeVoiceSettings({ copy, preferences, microphones, mic
             })),
             onValueChange: (value) => {
                 if (isOneOf(REALTIME_VOICE_INTELLIGENCE_LEVELS, value)) {
-                    onPreferencesChange({ ...preferences, intelligence: value });
+                    onPreferencesChange({
+                        ...preferences,
+                        intelligence: value,
+                        intelligenceChosen: true,
+                    });
                 }
             },
         },