@agent-native/core 0.98.0 → 0.98.1

package/corpus/core/src/client/composer/useRealtimeVoiceMode.tsx

@@ -15,7 +15,7 @@ import {
   SIDEBAR_STATE_CHANGE_EVENT,
   type AgentSidebarStateChangeDetail,
 } from "../agent-sidebar-state.js";
-import { agentNativePath, appPath } from "../api-path.js";
+import { agentNativePath } from "../api-path.js";
 import { readClientAppState, setClientAppState } from "../application-state.js";
 import { getBrowserTabId } from "../browser-tab-id.js";
 import { useT } from "../i18n.js";
@@ -33,10 +33,73 @@ import {
 } from "./RealtimeVoiceMode.js";
 
 const REALTIME_VOICE_STATE_KEY = "realtime-voice-session";
+const REALTIME_VOICE_PREFERENCES_KEY = "realtime-voice-prefs";
 const REALTIME_VOICE_REQUEST_SOURCE = "realtime-voice";
 const REALTIME_VOICE_SESSION_PATH = "/_agent-native/realtime-voice/session";
 const REALTIME_VOICE_TOOL_PATH = "/_agent-native/realtime-voice/tool";
+const REALTIME_VOICE_CAPABILITY_HEADER = "X-Agent-Native-Realtime-Capability";
 const REALTIME_VOICE_CONNECTION_TIMEOUT_MS = 15_000;
+const REALTIME_VOICE_MAX_TOOLS = 32;
+const REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES = 32_000;
+const REALTIME_VOICE_MAX_SESSION_BYTES = 64_000;
+const REALTIME_VOICE_TOOL_UPDATE_TIMEOUT_MS = 5_000;
+const REALTIME_VOICE_PRIORITY_TOOL_NAMES = [
+  "navigate",
+  "set-url-path",
+  "set-search-params",
+  "view-screen",
+  "tool-search",
+] as const;
+const REALTIME_VOICE_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/;
+const REALTIME_VOICE_MICROPHONE_STORAGE_KEY =
+  "agent-native:realtime-voice-microphone";
+
+export const REALTIME_VOICE_LANGUAGES = [
+  "auto",
+  "en",
+  "es",
+  "fr",
+  "de",
+  "it",
+  "pt",
+  "ja",
+  "ko",
+  "zh",
+] as const;
+export const REALTIME_VOICE_INTELLIGENCE_LEVELS = [
+  "instant",
+  "balanced",
+  "deep",
+] as const;
+export const REALTIME_VOICES = [
+  "marin",
+  "cedar",
+  "coral",
+  "sage",
+  "verse",
+  "alloy",
+  "ash",
+  "ballad",
+  "echo",
+  "shimmer",
+] as const;
+
+export type RealtimeVoiceLanguage = (typeof REALTIME_VOICE_LANGUAGES)[number];
+export type RealtimeVoiceIntelligence =
+  (typeof REALTIME_VOICE_INTELLIGENCE_LEVELS)[number];
+export type RealtimeVoice = (typeof REALTIME_VOICES)[number];
+
+export interface RealtimeVoicePreferences {
+  language: RealtimeVoiceLanguage;
+  intelligence: RealtimeVoiceIntelligence;
+  voice: RealtimeVoice;
+}
+
+export const DEFAULT_REALTIME_VOICE_PREFERENCES: RealtimeVoicePreferences = {
+  language: "auto",
+  intelligence: "instant",
+  voice: "marin",
+};
 
 export const REALTIME_VOICE_AUDIO_CONSTRAINTS: MediaTrackConstraints = {
   // Prefer the browser/OS-selected input instead of whichever physical device
@@ -55,6 +118,19 @@ export interface RealtimeVoiceToolResult {
   status: "completed" | "failed" | "approval_required";
   output: string;
   approvalKey?: string;
+  expandedTools?: RealtimeVoiceFunctionTool[];
+}
+
+export interface RealtimeVoiceSessionAnswer {
+  sdp: string;
+  capability?: string;
+}
+
+export interface RealtimeVoiceFunctionTool {
+  type: "function";
+  name: string;
+  description?: string;
+  parameters?: Record<string, unknown>;
 }
 
 export interface RealtimeVoiceModeApi {
@@ -63,17 +139,359 @@ export interface RealtimeVoiceModeApi {
   errorMessage: string | null;
   chatVisible: boolean;
   audioLevels: RealtimeVoiceAudioLevelStore;
+  preferences: RealtimeVoicePreferences;
+  microphones: readonly RealtimeVoiceMicrophone[];
+  microphoneDeviceId: string;
+  microphoneSwitching: boolean;
+  microphoneError: string | null;
+  voiceChangePending: boolean;
+  setLanguage: (language: RealtimeVoiceLanguage) => void;
+  setIntelligence: (intelligence: RealtimeVoiceIntelligence) => void;
+  setVoice: (voice: RealtimeVoice) => void;
+  setMicrophone: (deviceId: string) => Promise<void>;
   start: () => Promise<void>;
   end: () => void;
   toggleChat: () => void;
 }
 
+export interface RealtimeVoiceMicrophone {
+  deviceId: string;
+  label: string;
+}
+
+export function createRealtimeVoiceAudioConstraints(
+  deviceId = "default",
+  exact = false,
+): MediaTrackConstraints {
+  return {
+    ...REALTIME_VOICE_AUDIO_CONSTRAINTS,
+    deviceId:
+      deviceId === "default"
+        ? { ideal: "default" }
+        : exact
+          ? { exact: deviceId }
+          : { ideal: deviceId },
+  };
+}
+
+export async function replaceRealtimeVoiceMicrophone(options: {
+  mediaDevices: Pick<MediaDevices, "getUserMedia">;
+  peer: Pick<RTCPeerConnection, "getSenders">;
+  currentStream: MediaStream | null;
+  deviceId: string;
+}): Promise<MediaStream> {
+  const replacementStream = await options.mediaDevices.getUserMedia({
+    audio: createRealtimeVoiceAudioConstraints(options.deviceId, true),
+  });
+  try {
+    const replacementTrack = replacementStream.getAudioTracks()[0];
+    const sender = options.peer
+      .getSenders()
+      .find((candidate) => candidate.track?.kind === "audio");
+    if (!replacementTrack || !sender) {
+      throw new Error("No active microphone track is available.");
+    }
+
+    await sender.replaceTrack(replacementTrack);
+    for (const track of options.currentStream?.getTracks() ?? []) {
+      track.onended = null;
+      track.stop();
+    }
+    return replacementStream;
+  } catch (error) {
+    for (const track of replacementStream.getTracks()) track.stop();
+    throw error;
+  }
+}
+
+function readRealtimeVoiceMicrophoneId(): string {
+  if (typeof window === "undefined") return "default";
+  try {
+    return (
+      window.localStorage.getItem(REALTIME_VOICE_MICROPHONE_STORAGE_KEY) ||
+      "default"
+    );
+  } catch {
+    return "default";
+  }
+}
+
+function writeRealtimeVoiceMicrophoneId(deviceId: string): void {
+  try {
+    window.localStorage.setItem(
+      REALTIME_VOICE_MICROPHONE_STORAGE_KEY,
+      deviceId,
+    );
+  } catch {
+    // A remembered local device is optional and must never block voice mode.
+  }
+}
+
+function isOneOf<T extends readonly string[]>(
+  values: T,
+  value: unknown,
+): value is T[number] {
+  return typeof value === "string" && values.includes(value);
+}
+
+export function normalizeRealtimeVoicePreferences(
+  value: unknown,
+): RealtimeVoicePreferences {
+  const record =
+    value && typeof value === "object" && !Array.isArray(value)
+      ? (value as Record<string, unknown>)
+      : {};
+  return {
+    language: isOneOf(REALTIME_VOICE_LANGUAGES, record.language)
+      ? record.language
+      : DEFAULT_REALTIME_VOICE_PREFERENCES.language,
+    intelligence: isOneOf(
+      REALTIME_VOICE_INTELLIGENCE_LEVELS,
+      record.intelligence,
+    )
+      ? record.intelligence
+      : DEFAULT_REALTIME_VOICE_PREFERENCES.intelligence,
+    voice: isOneOf(REALTIME_VOICES, record.voice)
+      ? record.voice
+      : DEFAULT_REALTIME_VOICE_PREFERENCES.voice,
+  };
+}
+
+export function resolveRealtimeVoiceLanguage(
+  language: RealtimeVoiceLanguage,
+  browserLanguages: readonly string[] = [],
+): Exclude<RealtimeVoiceLanguage, "auto"> {
+  if (language !== "auto") return language;
+  for (const locale of browserLanguages) {
+    const primary = locale.trim().split("-")[0]?.toLowerCase();
+    if (isOneOf(REALTIME_VOICE_LANGUAGES, primary) && primary !== "auto") {
+      return primary;
+    }
+  }
+  return "en";
+}
+
+export function realtimeVoiceReasoningEffort(
+  intelligence: RealtimeVoiceIntelligence,
+): "minimal" | "low" | "medium" {
+  if (intelligence === "balanced") return "low";
+  if (intelligence === "deep") return "medium";
+  return "minimal";
+}
+
+export function createRealtimeVoicePreferenceUpdate(
+  preferences: RealtimeVoicePreferences,
+  options: {
+    browserLanguages?: readonly string[];
+    includeVoice?: boolean;
+  } = {},
+): Record<string, unknown> {
+  return {
+    type: "session.update",
+    session: {
+      type: "realtime",
+      reasoning: {
+        effort: realtimeVoiceReasoningEffort(preferences.intelligence),
+      },
+      audio: {
+        input: {
+          transcription: {
+            model: "gpt-4o-mini-transcribe",
+            language: resolveRealtimeVoiceLanguage(
+              preferences.language,
+              options.browserLanguages,
+            ),
+          },
+        },
+        ...(options.includeVoice
+          ? { output: { voice: preferences.voice } }
+          : {}),
+      },
+    },
+  };
+}
+
 export interface CompletedRealtimeVoiceTranscript {
   role: "user" | "assistant";
   text: string;
   providerId?: string;
 }
 
+export interface RealtimeVoiceTranscriptSequencer {
+  handle: (event: RealtimeServerEvent) => void;
+  reset: () => void;
+}
+
+interface SequencedRealtimeVoiceTranscript {
+  status: "pending" | "completed" | "skipped";
+  role?: CompletedRealtimeVoiceTranscript["role"];
+  transcript?: CompletedRealtimeVoiceTranscript;
+}
+
+/**
+ * Input transcription is produced by a separate ASR model and can finish after
+ * the assistant response. Reserve each message's position when OpenAI adds it
+ * to the conversation, then publish only the contiguous completed prefix.
+ */
+export function createRealtimeVoiceTranscriptSequencer(
+  publish: (transcript: CompletedRealtimeVoiceTranscript) => void,
+): RealtimeVoiceTranscriptSequencer {
+  const order: string[] = [];
+  const items = new Map<string, SequencedRealtimeVoiceTranscript>();
+  const settledItemIds = new Set<string>();
+  const receivedTranscriptIds = new Set<string>();
+
+  const reserve = (
+    id: string,
+    role?: CompletedRealtimeVoiceTranscript["role"],
+  ) => {
+    if (settledItemIds.has(id)) return;
+    const existing = items.get(id);
+    if (existing) {
+      existing.role ??= role;
+      return;
+    }
+    order.push(id);
+    items.set(id, { status: "pending", role });
+  };
+
+  const drain = () => {
+    while (order.length > 0) {
+      const id = order[0]!;
+      const entry = items.get(id);
+      if (!entry || entry.status === "pending") return;
+      order.shift();
+      items.delete(id);
+      settledItemIds.add(id);
+      if (entry.status === "completed" && entry.transcript) {
+        publish(entry.transcript);
+      }
+    }
+  };
+
+  const skip = (id: unknown) => {
+    if (typeof id !== "string" || !id) return;
+    reserve(id);
+    const entry = items.get(id)!;
+    if (entry.status === "pending") entry.status = "skipped";
+  };
+
+  return {
+    handle(event) {
+      if (
+        event.type === "conversation.item.added" ||
+        event.type === "conversation.item.created"
+      ) {
+        const item = event.item;
+        if (item && typeof item === "object") {
+          const record = item as Record<string, unknown>;
+          if (
+            record.type === "message" &&
+            (record.role === "user" || record.role === "assistant") &&
+            typeof record.id === "string"
+          ) {
+            reserve(record.id, record.role);
+          }
+        }
+      }
+
+      const transcript = extractCompletedRealtimeVoiceTranscript(event);
+      if (transcript) {
+        const transcriptId = transcript.providerId
+          ? `${transcript.role}:${transcript.providerId}`
+          : undefined;
+        if (transcriptId && receivedTranscriptIds.has(transcriptId)) return;
+        if (transcriptId) receivedTranscriptIds.add(transcriptId);
+        const itemId =
+          typeof event.item_id === "string" ? event.item_id : undefined;
+        if (!itemId) {
+          // Older providers may omit item_id. Match the first reserved slot for
+          // that role so one incomplete legacy event cannot strand the queue.
+          const reservedId = order.find((id) => {
+            const entry = items.get(id);
+            return (
+              entry?.status === "pending" && entry.role === transcript.role
+            );
+          });
+          if (reservedId) {
+            items.set(reservedId, {
+              status: "completed",
+              role: transcript.role,
+              transcript,
+            });
+          } else {
+            publish(transcript);
+          }
+        } else {
+          if (settledItemIds.has(itemId)) return;
+          reserve(itemId, transcript.role);
+          items.set(itemId, {
+            status: "completed",
+            role: transcript.role,
+            transcript,
+          });
+        }
+      } else if (
+        event.type === "conversation.item.input_audio_transcription.failed" ||
+        event.type === "conversation.item.deleted"
+      ) {
+        skip(event.item_id);
+      } else if (event.type === "response.done") {
+        const response = event.response;
+        const output =
+          response && typeof response === "object"
+            ? (response as { output?: unknown }).output
+            : undefined;
+        if (Array.isArray(output)) {
+          for (const item of output) {
+            if (!item || typeof item !== "object") continue;
+            const record = item as Record<string, unknown>;
+            if (record.type === "message") skip(record.id);
+          }
+        }
+      }
+      drain();
+    },
+    reset() {
+      order.length = 0;
+      items.clear();
+      settledItemIds.clear();
+      receivedTranscriptIds.clear();
+    },
+  };
+}
+
+export function createRealtimeVoiceGreetingEvent(): Record<string, unknown> {
+  return {
+    type: "response.create",
+    response: {
+      output_modalities: ["audio"],
+      instructions:
+        'Say exactly: "How can I help you?" Do not add anything else.',
+      // Do not set max_output_tokens here. Realtime counts assistant audio at
+      // roughly one token per 50 ms, so a text-sized cap can cut speech off.
+      // The exact-response instruction already keeps this greeting bounded.
+    },
+  };
+}
+
+export function createRealtimeVoiceGreetingStarter(
+  send: (event: Record<string, unknown>) => void,
+): { start: () => boolean; reset: () => void } {
+  let started = false;
+  return {
+    start() {
+      if (started) return false;
+      started = true;
+      send(createRealtimeVoiceGreetingEvent());
+      return true;
+    },
+    reset() {
+      started = false;
+    },
+  };
+}
+
 /**
  * Voice mode owns the chat only temporarily. Restore the captured transcript
  * when it is still the user's active thread (or the chat has no active thread)
@@ -127,6 +545,14 @@ export function isRealtimeVoiceAbortError(error: unknown): boolean {
   );
 }
 
+export function isRealtimeVoiceSetupRequiredError(error: unknown): boolean {
+  return Boolean(
+    error &&
+    typeof error === "object" &&
+    (error as { status?: unknown }).status === 409,
+  );
+}
+
 export function createRealtimeVoiceConnectionTimeout(
   onTimeout: () => void,
   timeoutMs = REALTIME_VOICE_CONNECTION_TIMEOUT_MS,
@@ -135,6 +561,24 @@ export function createRealtimeVoiceConnectionTimeout(
   return () => window.clearTimeout(timeout);
 }
 
+export function createRealtimeVoiceConnectionGate(
+  onTimeout: () => void,
+  timeoutMs = REALTIME_VOICE_CONNECTION_TIMEOUT_MS,
+): {
+  markTransportReady: () => void;
+  markSessionCreated: () => void;
+  cancel: () => void;
+} {
+  const cancel = createRealtimeVoiceConnectionTimeout(onTimeout, timeoutMs);
+  return {
+    // RTC connectivity alone is not enough: the session may never finish its
+    // Realtime handshake. Keep the deadline armed until session.created.
+    markTransportReady() {},
+    markSessionCreated: cancel,
+    cancel,
+  };
+}
+
 async function readErrorResponse(response: Response): Promise<string> {
   const raw = await response.text().catch(() => "");
   if (!raw) return response.statusText || `HTTP ${response.status}`;
@@ -146,10 +590,18 @@ async function readErrorResponse(response: Response): Promise<string> {
   }
 }
 
-export async function createRealtimeVoiceSession(
+interface RealtimeVoiceSessionOptions {
+  browserTabId?: string;
+  signal?: AbortSignal;
+  preferences?: RealtimeVoicePreferences;
+  browserLanguages?: readonly string[];
+}
+
+export async function createRealtimeVoiceSessionWithCapability(
   offerSdp: string,
-  options: { browserTabId?: string; signal?: AbortSignal } = {},
-): Promise<string> {
+  options: RealtimeVoiceSessionOptions = {},
+): Promise<RealtimeVoiceSessionAnswer> {
+  const preferences = options.preferences;
   const response = await fetch(agentNativePath(REALTIME_VOICE_SESSION_PATH), {
     method: "POST",
     credentials: "same-origin",
@@ -158,6 +610,16 @@ export async function createRealtimeVoiceSession(
       ...(options.browserTabId
         ? { "X-Agent-Native-Browser-Tab": options.browserTabId }
         : {}),
+      ...(preferences
+        ? {
+            "X-Agent-Native-Realtime-Language": resolveRealtimeVoiceLanguage(
+              preferences.language,
+              options.browserLanguages,
+            ),
+            "X-Agent-Native-Realtime-Intelligence": preferences.intelligence,
+            "X-Agent-Native-Realtime-Voice": preferences.voice,
+          }
+        : {}),
     },
     body: offerSdp,
     signal: options.signal,
@@ -168,7 +630,21 @@ export async function createRealtimeVoiceSession(
     (error as { status?: number }).status = response.status;
     throw error;
   }
-  return response.text();
+  const sdp = await response.text();
+  const capability = response.headers.get(REALTIME_VOICE_CAPABILITY_HEADER);
+  return {
+    sdp,
+    ...(capability ? { capability } : {}),
+  };
+}
+
+/** Backwards-compatible SDP-only session helper for existing client callers. */
+export async function createRealtimeVoiceSession(
+  offerSdp: string,
+  options: RealtimeVoiceSessionOptions = {},
+): Promise<string> {
+  return (await createRealtimeVoiceSessionWithCapability(offerSdp, options))
+    .sdp;
 }
 
 export async function executeRealtimeVoiceTool(input: {
@@ -177,6 +653,7 @@ export async function executeRealtimeVoiceTool(input: {
   callId: string;
   sessionId?: string;
   browserTabId?: string;
+  capability?: string;
   signal?: AbortSignal;
 }): Promise<RealtimeVoiceToolResult> {
   const response = await fetch(agentNativePath(REALTIME_VOICE_TOOL_PATH), {
@@ -187,6 +664,9 @@ export async function executeRealtimeVoiceTool(input: {
       ...(input.browserTabId
         ? { "X-Agent-Native-Browser-Tab": input.browserTabId }
         : {}),
+      ...(input.capability
+        ? { [REALTIME_VOICE_CAPABILITY_HEADER]: input.capability }
+        : {}),
     },
     body: JSON.stringify({
       name: input.name,
@@ -203,6 +683,264 @@ export async function executeRealtimeVoiceTool(input: {
   return (await response.json()) as RealtimeVoiceToolResult;
 }
 
+function jsonByteLength(value: unknown): number {
+  return new TextEncoder().encode(JSON.stringify(value)).byteLength;
+}
+
+function normalizeRealtimeVoiceFunctionTool(
+  value: unknown,
+): RealtimeVoiceFunctionTool | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const record = value as Record<string, unknown>;
+  if (
+    record.type !== "function" ||
+    typeof record.name !== "string" ||
+    !REALTIME_VOICE_TOOL_NAME.test(record.name)
+  ) {
+    return null;
+  }
+  if (
+    record.description !== undefined &&
+    typeof record.description !== "string"
+  ) {
+    return null;
+  }
+  if (
+    record.parameters !== undefined &&
+    (!record.parameters ||
+      typeof record.parameters !== "object" ||
+      Array.isArray(record.parameters))
+  ) {
+    return null;
+  }
+  try {
+    if (jsonByteLength(record) > REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES) {
+      return null;
+    }
+    return JSON.parse(JSON.stringify(record)) as RealtimeVoiceFunctionTool;
+  } catch {
+    return null;
+  }
+}
+
+export function extractRealtimeVoiceSessionTools(
+  event: RealtimeServerEvent,
+): RealtimeVoiceFunctionTool[] | null {
+  if (event.type !== "session.created" && event.type !== "session.updated") {
+    return null;
+  }
+  const session = event.session;
+  if (!session || typeof session !== "object") return null;
+  const tools = (session as { tools?: unknown }).tools;
+  if (!Array.isArray(tools)) return null;
+  return tools
+    .map(normalizeRealtimeVoiceFunctionTool)
+    .filter((tool): tool is RealtimeVoiceFunctionTool => Boolean(tool));
+}
+
+function createRealtimeVoiceToolManifestUpdate(
+  tools: RealtimeVoiceFunctionTool[],
+  eventId?: string,
+): Record<string, unknown> {
+  return {
+    type: "session.update",
+    ...(eventId ? { event_id: eventId } : {}),
+    session: {
+      type: "realtime",
+      tools,
+      tool_choice: "auto",
+    },
+  };
+}
+
+/**
+ * Merge search-discovered schemas into the live manifest. Pinned navigation
+ * and discovery tools stay available, then the newest discoveries take the
+ * remaining slots ahead of lower-priority tools from the original manifest.
+ */
+export function mergeRealtimeVoiceToolManifest(
+  currentTools: readonly RealtimeVoiceFunctionTool[],
+  expandedTools: readonly RealtimeVoiceFunctionTool[],
+): RealtimeVoiceFunctionTool[] {
+  const current = new Map<string, RealtimeVoiceFunctionTool>();
+  for (const value of currentTools) {
+    const tool = normalizeRealtimeVoiceFunctionTool(value);
+    if (tool) current.set(tool.name, tool);
+  }
+  const expanded = new Map<string, RealtimeVoiceFunctionTool>();
+  for (const value of expandedTools) {
+    const tool = normalizeRealtimeVoiceFunctionTool(value);
+    if (tool) expanded.set(tool.name, tool);
+  }
+
+  const candidates: RealtimeVoiceFunctionTool[] = [];
+  const seen = new Set<string>();
+  const add = (tool: RealtimeVoiceFunctionTool | undefined) => {
+    if (!tool || seen.has(tool.name)) return;
+    seen.add(tool.name);
+    candidates.push(tool);
+  };
+  for (const name of REALTIME_VOICE_PRIORITY_TOOL_NAMES) {
+    add(expanded.get(name) ?? current.get(name));
+  }
+  for (const tool of expanded.values()) add(tool);
+  for (const tool of current.values()) add(tool);
+
+  const packed: RealtimeVoiceFunctionTool[] = [];
+  for (const tool of candidates) {
+    if (packed.length >= REALTIME_VOICE_MAX_TOOLS) break;
+    const candidate = [...packed, tool];
+    if (
+      jsonByteLength(
+        createRealtimeVoiceToolManifestUpdate(
+          candidate,
+          "realtime_tool_manifest_999999999",
+        ),
+      ) <= REALTIME_VOICE_MAX_SESSION_BYTES
+    ) {
+      packed.push(tool);
+    }
+  }
+  return packed;
+}
+
+function createRealtimeVoiceToolResultEvent(
+  result: RealtimeVoiceToolResult,
+): Record<string, unknown> {
+  const modelResult = {
+    callId: result.callId,
+    status: result.status,
+    output: result.output,
+    ...(result.approvalKey ? { approvalKey: result.approvalKey } : {}),
+  };
+  return {
+    type: "conversation.item.create",
+    item: {
+      type: "function_call_output",
+      call_id: result.callId,
+      output: JSON.stringify(modelResult),
+    },
+  };
+}
+
+export interface RealtimeVoiceToolManifestCoordinator {
+  enqueue: (result: RealtimeVoiceToolResult) => void;
+  setSessionTools: (tools: readonly RealtimeVoiceFunctionTool[]) => void;
+  handleError: (eventId: string | undefined, message?: string) => boolean;
+  reset: () => void;
+  getTools: () => readonly RealtimeVoiceFunctionTool[];
+}
+
+/**
+ * Realtime treats `session.update.tools` as a full replacement. Serialize
+ * discovery updates and wait for a confirming `session.updated` manifest
+ * before returning the search result and allowing the next model response.
+ */
+export function createRealtimeVoiceToolManifestCoordinator(
+  send: (event: Record<string, unknown>) => void,
+  timeoutMs = REALTIME_VOICE_TOOL_UPDATE_TIMEOUT_MS,
+): RealtimeVoiceToolManifestCoordinator {
+  let currentTools: RealtimeVoiceFunctionTool[] = [];
+  const queue: RealtimeVoiceToolResult[] = [];
+  let updateSequence = 0;
+  let active:
+    | {
+        result: RealtimeVoiceToolResult;
+        expectedNames: string[];
+        eventId: string;
+        timer: ReturnType<typeof setTimeout>;
+      }
+    | undefined;
+
+  const finish = (failureMessage?: string) => {
+    if (!active) return;
+    const { result, timer } = active;
+    clearTimeout(timer);
+    active = undefined;
+    send(
+      createRealtimeVoiceToolResultEvent(
+        failureMessage
+          ? {
+              callId: result.callId,
+              status: "failed",
+              output: `${failureMessage} The discovered tools were not added to this voice session. Original tool result: ${result.output}`,
+            }
+          : result,
+      ),
+    );
+    send({ type: "response.create" });
+    drain();
+  };
+
+  const drain = () => {
+    if (active) return;
+    const result = queue.shift();
+    if (!result) return;
+    const expanded = Array.isArray(result.expandedTools)
+      ? result.expandedTools
+          .map(normalizeRealtimeVoiceFunctionTool)
+          .filter((tool): tool is RealtimeVoiceFunctionTool => Boolean(tool))
+      : [];
+    if (result.status !== "completed" || expanded.length === 0) {
+      send(createRealtimeVoiceToolResultEvent(result));
+      send({ type: "response.create" });
+      drain();
+      return;
+    }
+
+    const merged = mergeRealtimeVoiceToolManifest(currentTools, expanded);
+    const mergedNames = new Set(merged.map((tool) => tool.name));
+    const expectedNames = expanded
+      .map((tool) => tool.name)
+      .filter((name) => mergedNames.has(name));
+    if (expectedNames.length === 0) {
+      send(createRealtimeVoiceToolResultEvent(result));
+      send({ type: "response.create" });
+      drain();
+      return;
+    }
+
+    const eventId = `realtime_tool_manifest_${++updateSequence}`;
+    active = {
+      result,
+      expectedNames,
+      eventId,
+      timer: setTimeout(
+        () => finish("Timed out while installing the discovered tools."),
+        timeoutMs,
+      ),
+    };
+    send(createRealtimeVoiceToolManifestUpdate(merged, eventId));
+  };
+
+  return {
+    enqueue(result) {
+      queue.push(result);
+      drain();
+    },
+    setSessionTools(tools) {
+      currentTools = mergeRealtimeVoiceToolManifest([], tools);
+      if (!active) return;
+      const names = new Set(currentTools.map((tool) => tool.name));
+      if (active.expectedNames.every((name) => names.has(name))) finish();
+    },
+    handleError(eventId, message) {
+      if (!active || !eventId || active.eventId !== eventId) return false;
+      finish(message || "The provider rejected the discovered tool update.");
+      return true;
+    },
+    reset() {
+      if (active) clearTimeout(active.timer);
+      active = undefined;
+      queue.length = 0;
+      currentTools = [];
+    },
+    getTools() {
+      return currentTools;
+    },
+  };
+}
+
 export function extractRealtimeVoiceFunctionCalls(
   event: RealtimeServerEvent,
 ): Array<{ name: string; callId: string; argumentsText: string }> {
@@ -269,19 +1007,6 @@ function openOpenAiKeySettings(): void {
   );
 }
 
-function openMicrophoneSettings(): void {
-  if (typeof window === "undefined") return;
-  window.location.assign(`${appPath("/settings")}#voice`);
-}
-
-export function endRealtimeVoiceBeforeNavigation(
-  end: () => void,
-  navigate: () => void,
-): void {
-  end();
-  window.setTimeout(navigate, 0);
-}
-
 export function listenForRealtimeVoicePageHide(
   cleanup: () => void,
 ): () => void {
@@ -304,7 +1029,49 @@ function voiceCopy(t: ReturnType<typeof useT>): RealtimeVoiceModeCopy {
     showChat: t("agentPanel.voiceMode.showChat"),
     hideChat: t("agentPanel.voiceMode.hideChat"),
     endVoiceMode: t("agentPanel.voiceMode.end"),
-    microphoneSettings: t("agentPanel.voiceMode.microphoneSettings"),
+    voiceSettings: t("agentPanel.voiceMode.voiceSettings"),
+    settings: {
+      language: t("agentPanel.voiceMode.settings.language"),
+      autoLanguage: t("agentPanel.voiceMode.settings.autoLanguage"),
+      languages: {
+        en: t("agentPanel.voiceMode.settings.languages.en"),
+        es: t("agentPanel.voiceMode.settings.languages.es"),
+        fr: t("agentPanel.voiceMode.settings.languages.fr"),
+        de: t("agentPanel.voiceMode.settings.languages.de"),
+        it: t("agentPanel.voiceMode.settings.languages.it"),
+        pt: t("agentPanel.voiceMode.settings.languages.pt"),
+        ja: t("agentPanel.voiceMode.settings.languages.ja"),
+        ko: t("agentPanel.voiceMode.settings.languages.ko"),
+        zh: t("agentPanel.voiceMode.settings.languages.zh"),
+      },
+      intelligence: t("agentPanel.voiceMode.settings.intelligence"),
+      intelligenceLevels: {
+        instant: t("agentPanel.voiceMode.settings.intelligenceLevels.instant"),
+        balanced: t(
+          "agentPanel.voiceMode.settings.intelligenceLevels.balanced",
+        ),
+        deep: t("agentPanel.voiceMode.settings.intelligenceLevels.deep"),
+      },
+      voiceStyle: t("agentPanel.voiceMode.settings.voiceStyle"),
+      microphone: t("agentPanel.voiceMode.settings.microphone"),
+      defaultMicrophone: t("agentPanel.voiceMode.settings.defaultMicrophone"),
+      microphoneSwitchFailed: t(
+        "agentPanel.voiceMode.settings.microphoneSwitchFailed",
+      ),
+      voiceChangePending: t("agentPanel.voiceMode.settings.voiceChangePending"),
+      voiceDescriptions: {
+        marin: t("agentPanel.voiceMode.settings.voiceDescriptions.marin"),
+        cedar: t("agentPanel.voiceMode.settings.voiceDescriptions.cedar"),
+        coral: t("agentPanel.voiceMode.settings.voiceDescriptions.coral"),
+        sage: t("agentPanel.voiceMode.settings.voiceDescriptions.sage"),
+        verse: t("agentPanel.voiceMode.settings.voiceDescriptions.verse"),
+        alloy: t("agentPanel.voiceMode.settings.voiceDescriptions.alloy"),
+        ash: t("agentPanel.voiceMode.settings.voiceDescriptions.ash"),
+        ballad: t("agentPanel.voiceMode.settings.voiceDescriptions.ballad"),
+        echo: t("agentPanel.voiceMode.settings.voiceDescriptions.echo"),
+        shimmer: t("agentPanel.voiceMode.settings.voiceDescriptions.shimmer"),
+      },
+    },
     status: {
       connecting: t("agentPanel.voiceMode.status.connecting"),
       listening: t("agentPanel.voiceMode.status.listening"),
@@ -318,6 +1085,7 @@ function voiceCopy(t: ReturnType<typeof useT>): RealtimeVoiceModeCopy {
       responseFailed: t("agentPanel.voiceMode.errors.responseFailed"),
       sessionFailed: t("agentPanel.voiceMode.errors.sessionFailed"),
       channelDisconnected: t("agentPanel.voiceMode.errors.channelDisconnected"),
+      connectionTimedOut: t("agentPanel.voiceMode.errors.connectionTimedOut"),
       connectionFailed: t("agentPanel.voiceMode.errors.connectionFailed"),
       offerFailed: t("agentPanel.voiceMode.errors.offerFailed"),
     },
@@ -336,8 +1104,20 @@ function useRealtimeVoiceModeController(
   const [state, setState] = useState<"idle" | RealtimeVoiceModeState>("idle");
   const [error, setError] = useState<string | null>(null);
   const [chatVisible, setChatVisible] = useState(false);
+  const [preferences, setPreferences] = useState(
+    DEFAULT_REALTIME_VOICE_PREFERENCES,
+  );
+  const [voiceChangePending, setVoiceChangePending] = useState(false);
+  const [microphones, setMicrophones] = useState<RealtimeVoiceMicrophone[]>([]);
+  const [microphoneDeviceId, setMicrophoneDeviceId] = useState(
+    readRealtimeVoiceMicrophoneId,
+  );
+  const [microphoneSwitching, setMicrophoneSwitching] = useState(false);
+  const [microphoneError, setMicrophoneError] = useState<string | null>(null);
   const [audioLevels] = useState(createRealtimeVoiceAudioLevelStore);
   const stateRef = useRef(state);
+  const preferencesRef = useRef(preferences);
+  const hasOutputAudioRef = useRef(false);
   const peerRef = useRef<RTCPeerConnection | null>(null);
   const channelRef = useRef<RTCDataChannel | null>(null);
   const streamRef = useRef<MediaStream | null>(null);
@@ -352,19 +1132,56 @@ function useRealtimeVoiceModeController(
   const meterFrameRef = useRef<number | null>(null);
   const lastMeterSampleRef = useRef(0);
   const abortRef = useRef<AbortController | null>(null);
-  const cancelConnectionTimeoutRef = useRef<(() => void) | null>(null);
+  const connectionGateRef = useRef<ReturnType<
+    typeof createRealtimeVoiceConnectionGate
+  > | null>(null);
   const handledCallsRef = useRef(new Set<string>());
   const sessionIdRef = useRef<string | undefined>(undefined);
+  const capabilityRef = useRef<string | undefined>(undefined);
+  const transportGenerationRef = useRef(0);
   const startedAtRef = useRef<string | undefined>(undefined);
   const lastUserTextRef = useRef("");
   const lastAssistantTextRef = useRef("");
   const transcriptThreadIdRef = useRef<string | undefined>(undefined);
   const transcriptSequenceRef = useRef(0);
+  const preferencesHydratedRef = useRef(false);
+  const preferencesEditedRef = useRef(false);
+  const preferenceWriteChainRef = useRef<Promise<void>>(Promise.resolve());
+  const toolManifestCoordinator = useMemo(
+    () =>
+      createRealtimeVoiceToolManifestCoordinator((event) => {
+        sendDataChannelEvent(channelRef.current, event);
+      }),
+    [],
+  );
+  const switchMicrophoneRef = useRef<
+    (deviceId: string, force?: boolean) => Promise<void>
+  >(async () => undefined);
 
   useEffect(() => {
     stateRef.current = state;
   }, [state]);
 
+  useEffect(() => {
+    preferencesRef.current = preferences;
+  }, [preferences]);
+
+  const hydratePreferences = useCallback(async () => {
+    if (preferencesHydratedRef.current) return;
+    try {
+      const stored = await readClientAppState(REALTIME_VOICE_PREFERENCES_KEY);
+      if (!preferencesEditedRef.current && stored != null) {
+        const next = normalizeRealtimeVoicePreferences(stored);
+        preferencesRef.current = next;
+        setPreferences(next);
+      }
+    } catch {
+      // Preferences are optional; keep safe defaults when storage is unavailable.
+    } finally {
+      preferencesHydratedRef.current = true;
+    }
+  }, []);
+
   const syncAppState = useCallback(
     (nextState: "idle" | RealtimeVoiceModeState) => {
       const value =
@@ -377,6 +1194,7 @@ function useRealtimeVoiceModeController(
               startedAt: startedAtRef.current,
               sessionId: sessionIdRef.current,
               browserTabId,
+              preferences: preferencesRef.current,
               lastUserText: lastUserTextRef.current || undefined,
               lastAssistantText: lastAssistantTextRef.current || undefined,
             };
@@ -396,6 +1214,95 @@ function useRealtimeVoiceModeController(
     [syncAppState],
   );
 
+  const savePreferences = useCallback((next: RealtimeVoicePreferences) => {
+    preferencesEditedRef.current = true;
+    preferencesHydratedRef.current = true;
+    preferencesRef.current = next;
+    setPreferences(next);
+    preferenceWriteChainRef.current = preferenceWriteChainRef.current
+      .catch(() => undefined)
+      .then(async () => {
+        await setClientAppState(REALTIME_VOICE_PREFERENCES_KEY, next, {
+          requestSource: REALTIME_VOICE_REQUEST_SOURCE,
+        });
+      })
+      .catch(() => undefined);
+  }, []);
+
+  const updateLivePreferences = useCallback(
+    (next: RealtimeVoicePreferences, includeVoice = false) => {
+      sendDataChannelEvent(
+        channelRef.current,
+        createRealtimeVoicePreferenceUpdate(next, {
+          browserLanguages:
+            typeof navigator === "undefined" ? [] : navigator.languages,
+          includeVoice,
+        }),
+      );
+    },
+    [],
+  );
+
+  const setLanguage = useCallback(
+    (language: RealtimeVoiceLanguage) => {
+      const next = { ...preferencesRef.current, language };
+      savePreferences(next);
+      updateLivePreferences(next);
+    },
+    [savePreferences, updateLivePreferences],
+  );
+
+  const setIntelligence = useCallback(
+    (intelligence: RealtimeVoiceIntelligence) => {
+      const next = { ...preferencesRef.current, intelligence };
+      savePreferences(next);
+      updateLivePreferences(next);
+    },
+    [savePreferences, updateLivePreferences],
+  );
+
+  const setVoice = useCallback(
+    (voice: RealtimeVoice) => {
+      const next = { ...preferencesRef.current, voice };
+      savePreferences(next);
+      if (hasOutputAudioRef.current) {
+        setVoiceChangePending(true);
+        updateLivePreferences(next);
+        return;
+      }
+      setVoiceChangePending(false);
+      updateLivePreferences(next, true);
+    },
+    [savePreferences, updateLivePreferences],
+  );
+
+  const refreshMicrophones = useCallback(async (): Promise<
+    RealtimeVoiceMicrophone[] | null
+  > => {
+    const mediaDevices = navigator.mediaDevices;
+    if (!mediaDevices?.enumerateDevices) return null;
+    try {
+      const devices = await mediaDevices.enumerateDevices();
+      const inputs = devices
+        .filter(
+          (device) =>
+            device.kind === "audioinput" &&
+            device.deviceId &&
+            device.deviceId !== "default" &&
+            device.deviceId !== "communications",
+        )
+        .map((device, index) => ({
+          deviceId: device.deviceId,
+          label: device.label || `Microphone ${index + 1}`,
+        }));
+      setMicrophones(inputs);
+      return inputs;
+    } catch {
+      // Enumeration is progressive enhancement; system default remains usable.
+      return null;
+    }
+  }, []);
+
   const startMeterLoop = useCallback(() => {
     if (meterFrameRef.current !== null) return;
     const sample = (timestamp: number) => {
@@ -467,16 +1374,69 @@ function useRealtimeVoiceModeController(
     [startMeterLoop],
   );
 
+  const setMicrophone = useCallback(
+    async (deviceId: string, force = false) => {
+      if ((!force && deviceId === microphoneDeviceId) || microphoneSwitching) {
+        return;
+      }
+      const mediaDevices = navigator.mediaDevices;
+      const peer = peerRef.current;
+      if (!mediaDevices?.getUserMedia || !peer) return;
+
+      setMicrophoneSwitching(true);
+      setMicrophoneError(null);
+      try {
+        const replacementStream = await replaceRealtimeVoiceMicrophone({
+          mediaDevices,
+          peer,
+          currentStream: streamRef.current,
+          deviceId,
+        });
+        const replacementTrack = replacementStream.getAudioTracks()[0];
+        streamRef.current = replacementStream;
+        if (replacementTrack) {
+          replacementTrack.onended = () => {
+            void switchMicrophoneRef.current("default", true);
+          };
+        }
+        attachAudioMeter(streamRef.current, "input");
+        writeRealtimeVoiceMicrophoneId(deviceId);
+        setMicrophoneDeviceId(deviceId);
+        await refreshMicrophones();
+      } catch {
+        setMicrophoneError(
+          copy?.settings.microphoneSwitchFailed ??
+            "Could not switch microphones. Your current microphone is still active.",
+        );
+      } finally {
+        setMicrophoneSwitching(false);
+      }
+    },
+    [
+      attachAudioMeter,
+      copy,
+      microphoneDeviceId,
+      microphoneSwitching,
+      refreshMicrophones,
+    ],
+  );
+  switchMicrophoneRef.current = setMicrophone;
+
   const cleanupTransport = useCallback(() => {
-    cancelConnectionTimeoutRef.current?.();
-    cancelConnectionTimeoutRef.current = null;
+    connectionGateRef.current?.cancel();
+    connectionGateRef.current = null;
+    transportGenerationRef.current += 1;
+    capabilityRef.current = undefined;
     abortRef.current?.abort();
     abortRef.current = null;
     channelRef.current?.close();
     channelRef.current = null;
     peerRef.current?.close();
     peerRef.current = null;
-    for (const track of streamRef.current?.getTracks() ?? []) track.stop();
+    for (const track of streamRef.current?.getTracks() ?? []) {
+      track.onended = null;
+      track.stop();
+    }
     streamRef.current = null;
     if (audioRef.current) {
       audioRef.current.pause();
@@ -501,7 +1461,8 @@ function useRealtimeVoiceModeController(
     if (audioContext) void audioContext.close().catch(() => undefined);
     audioLevels.reset();
     handledCallsRef.current.clear();
-  }, [audioLevels]);
+    toolManifestCoordinator.reset();
+  }, [audioLevels, toolManifestCoordinator]);
 
   const fail = useCallback(
     (message: string, options?: { openKeySettings?: boolean }) => {
@@ -517,6 +1478,8 @@ function useRealtimeVoiceModeController(
     async (call: { name: string; callId: string; argumentsText: string }) => {
       if (handledCallsRef.current.has(call.callId)) return;
       handledCallsRef.current.add(call.callId);
+      const transportGeneration = transportGenerationRef.current;
+      const transportChannel = channelRef.current;
       transition("working");
       let result: RealtimeVoiceToolResult;
       try {
@@ -527,6 +1490,7 @@ function useRealtimeVoiceModeController(
           callId: call.callId,
           sessionId: sessionIdRef.current,
           browserTabId,
+          capability: capabilityRef.current,
           signal: abortRef.current?.signal,
         });
       } catch (toolError) {
@@ -536,24 +1500,21 @@ function useRealtimeVoiceModeController(
           output: errorMessage(toolError),
         };
       }
-      sendDataChannelEvent(channelRef.current, {
-        type: "conversation.item.create",
-        item: {
-          type: "function_call_output",
-          call_id: call.callId,
-          output: JSON.stringify(result),
-        },
-      });
-      sendDataChannelEvent(channelRef.current, { type: "response.create" });
+      if (
+        transportGeneration !== transportGenerationRef.current ||
+        channelRef.current !== transportChannel
+      ) {
+        return;
+      }
+      toolManifestCoordinator.enqueue(result);
     },
-    [browserTabId, transition],
+    [browserTabId, toolManifestCoordinator, transition],
   );
 
-  const persistCompletedTranscript = useCallback(
-    (event: RealtimeServerEvent) => {
-      const transcript = extractCompletedRealtimeVoiceTranscript(event);
+  const publishCompletedTranscript = useCallback(
+    (transcript: CompletedRealtimeVoiceTranscript) => {
       const threadId = transcriptThreadIdRef.current;
-      if (!transcript || !threadId) return;
+      if (!threadId) return;
       const sessionIdentity =
         sessionIdRef.current ?? startedAtRef.current ?? "pending";
       const providerIdentity =
@@ -569,23 +1530,46 @@ function useRealtimeVoiceModeController(
     [],
   );
 
+  const transcriptSequencer = useMemo(
+    () => createRealtimeVoiceTranscriptSequencer(publishCompletedTranscript),
+    [publishCompletedTranscript],
+  );
+  const greetingStarter = useMemo(
+    () =>
+      createRealtimeVoiceGreetingStarter((event) => {
+        sendDataChannelEvent(channelRef.current, event);
+      }),
+    [],
+  );
+
   const handleServerEvent = useCallback(
     (event: RealtimeServerEvent) => {
+      transcriptSequencer.handle(event);
+      const sessionTools = extractRealtimeVoiceSessionTools(event);
+      if (sessionTools) toolManifestCoordinator.setSessionTools(sessionTools);
       if (event.type === "session.created") {
+        connectionGateRef.current?.markSessionCreated();
+        connectionGateRef.current = null;
         const session = event.session;
         if (session && typeof session === "object") {
           const id = (session as { id?: unknown }).id;
           if (typeof id === "string") sessionIdRef.current = id;
         }
-        transition("listening");
+        updateLivePreferences(preferencesRef.current, true);
+        greetingStarter.start();
+        transition("working");
       } else if (event.type === "input_audio_buffer.speech_started") {
         transition("listening");
       } else if (event.type === "input_audio_buffer.speech_stopped") {
         transition("working");
       } else if (event.type === "response.created") {
+        // From this point onward the response is committed to audio output, so
+        // changing voices risks violating Realtime's per-session voice lock.
+        hasOutputAudioRef.current = true;
         lastAssistantTextRef.current = "";
         transition("working");
       } else if (event.type === "response.output_audio_transcript.delta") {
+        hasOutputAudioRef.current = true;
         if (typeof event.delta === "string") {
           lastAssistantTextRef.current += event.delta;
         }
@@ -594,7 +1578,6 @@ function useRealtimeVoiceModeController(
         if (typeof event.transcript === "string") {
           lastAssistantTextRef.current = event.transcript;
         }
-        persistCompletedTranscript(event);
         syncAppState("speaking");
       } else if (
         event.type === "conversation.item.input_audio_transcription.completed"
@@ -602,7 +1585,6 @@ function useRealtimeVoiceModeController(
         if (typeof event.transcript === "string") {
           lastUserTextRef.current = event.transcript;
         }
-        persistCompletedTranscript(event);
         syncAppState("working");
       } else if (event.type === "response.done") {
         const response = event.response;
@@ -625,6 +1607,18 @@ function useRealtimeVoiceModeController(
             : typeof detail === "string"
               ? detail
               : "";
+        const expansionEventId =
+          detail && typeof detail === "object"
+            ? ((detail as { event_id?: unknown }).event_id ?? event.event_id)
+            : event.event_id;
+        if (
+          toolManifestCoordinator.handleError(
+            typeof expansionEventId === "string" ? expansionEventId : undefined,
+            message,
+          )
+        ) {
+          return;
+        }
         fail(
           message ||
             copy?.errors.sessionFailed ||
@@ -642,10 +1636,13 @@ function useRealtimeVoiceModeController(
     [
       copy,
       fail,
+      greetingStarter,
       handleFunctionCall,
-      persistCompletedTranscript,
       syncAppState,
+      transcriptSequencer,
       transition,
+      toolManifestCoordinator,
+      updateLivePreferences,
     ],
   );
 
@@ -666,42 +1663,62 @@ function useRealtimeVoiceModeController(
     lastUserTextRef.current = "";
     lastAssistantTextRef.current = "";
     sessionIdRef.current = undefined;
+    hasOutputAudioRef.current = false;
+    greetingStarter.reset();
+    setVoiceChangePending(false);
     transcriptThreadIdRef.current =
       realtimeVoiceTranscriptRegistry.activeThreadId();
     transcriptSequenceRef.current = 0;
+    transcriptSequencer.reset();
     transition("connecting");
     setChatVisible(false);
     window.dispatchEvent(new Event("agent-panel:close"));
 
+    await hydratePreferences();
+    if ((stateRef.current as string) !== "connecting") return;
+
     const abortController = new AbortController();
     abortRef.current = abortController;
     const isCurrentAttempt = () =>
       abortRef.current === abortController && !abortController.signal.aborted;
-    const markConnected = () => {
+    const connectionGate = createRealtimeVoiceConnectionGate(() => {
       if (!isCurrentAttempt()) return;
-      cancelConnectionTimeoutRef.current?.();
-      cancelConnectionTimeoutRef.current = null;
-      transition("listening");
-    };
-    cancelConnectionTimeoutRef.current = createRealtimeVoiceConnectionTimeout(
-      () => {
-        if (!isCurrentAttempt()) return;
-        fail(
-          copy?.errors.connectionFailed ??
-            "The realtime voice connection timed out.",
-        );
-      },
-    );
+      fail(
+        copy?.errors.connectionTimedOut ??
+          "The realtime voice connection timed out.",
+      );
+    });
+    connectionGateRef.current = connectionGate;
     try {
       const stream = await navigator.mediaDevices.getUserMedia({
-        audio: REALTIME_VOICE_AUDIO_CONSTRAINTS,
+        audio: createRealtimeVoiceAudioConstraints(
+          readRealtimeVoiceMicrophoneId(),
+        ),
       });
       if (!isCurrentAttempt()) {
         for (const track of stream.getTracks()) track.stop();
         return;
       }
       streamRef.current = stream;
+      for (const track of stream.getAudioTracks()) {
+        track.onended = () => {
+          void switchMicrophoneRef.current("default", true);
+        };
+      }
       attachAudioMeter(stream, "input");
+      void refreshMicrophones().then((inputs) => {
+        const selected = readRealtimeVoiceMicrophoneId();
+        if (
+          inputs &&
+          selected !== "default" &&
+          !inputs.some((device) => device.deviceId === selected)
+        ) {
+          // The persisted device was only an ideal preference, so getUserMedia
+          // already selected a usable fallback for this new session.
+          writeRealtimeVoiceMicrophoneId("default");
+          setMicrophoneDeviceId("default");
+        }
+      });
 
       const peer = new RTCPeerConnection();
       peerRef.current = peer;
@@ -720,7 +1737,7 @@ function useRealtimeVoiceModeController(
 
       const channel = peer.createDataChannel("oai-events");
       channelRef.current = channel;
-      channel.onopen = markConnected;
+      channel.onopen = connectionGate.markTransportReady;
       channel.onmessage = (messageEvent) => {
         if (!isCurrentAttempt()) return;
         try {
@@ -738,7 +1755,9 @@ function useRealtimeVoiceModeController(
       };
       peer.onconnectionstatechange = () => {
         if (!isCurrentAttempt()) return;
-        if (peer.connectionState === "connected") markConnected();
+        if (peer.connectionState === "connected") {
+          connectionGate.markTransportReady();
+        }
         if (peer.connectionState === "failed") {
           fail(
             copy?.errors.connectionFailed ??
@@ -757,12 +1776,15 @@ function useRealtimeVoiceModeController(
             "The browser did not create an audio offer.",
         );
       }
-      const answerSdp = await createRealtimeVoiceSession(offer.sdp, {
+      const answer = await createRealtimeVoiceSessionWithCapability(offer.sdp, {
         browserTabId,
         signal: abortController.signal,
+        preferences: preferencesRef.current,
+        browserLanguages: navigator.languages,
       });
       if (!isCurrentAttempt()) return;
-      await peer.setRemoteDescription({ type: "answer", sdp: answerSdp });
+      capabilityRef.current = answer.capability;
+      await peer.setRemoteDescription({ type: "answer", sdp: answer.sdp });
     } catch (startError) {
       // Ending a connection aborts the SDP request by design. A superseded
       // attempt must never resurrect the dock or surface that cancellation as
@@ -774,15 +1796,36 @@ function useRealtimeVoiceModeController(
       ) {
         return;
       }
+      if (isRealtimeVoiceSetupRequiredError(startError)) {
+        // Credentials can disappear after the entry-point status check. Treat
+        // the authoritative setup response as a gate, not as an active voice
+        // session failure: clean up the mic/RTC attempt, reopen chat, and
+        // refresh both the chat and voice provider setup surfaces.
+        cleanupTransport();
+        setError(null);
+        startedAtRef.current = undefined;
+        sessionIdRef.current = undefined;
+        transcriptThreadIdRef.current = undefined;
+        transition("idle");
+        setChatVisible(true);
+        window.dispatchEvent(new Event("agent-panel:open"));
+        window.dispatchEvent(new Event("agent-engine:configured-changed"));
+        return;
+      }
       const status = (startError as { status?: unknown })?.status;
       fail(errorMessage(startError), { openKeySettings: status === 400 });
     }
   }, [
     attachAudioMeter,
     browserTabId,
+    cleanupTransport,
     copy,
     fail,
+    greetingStarter,
     handleServerEvent,
+    hydratePreferences,
+    refreshMicrophones,
+    transcriptSequencer,
     transition,
   ]);
 
@@ -821,6 +1864,26 @@ function useRealtimeVoiceModeController(
     window.dispatchEvent(new Event("agent-panel:toggle"));
   }, []);
 
+  useEffect(() => {
+    const mediaDevices = navigator.mediaDevices;
+    if (!mediaDevices?.addEventListener) return;
+    const handleDeviceChange = () => {
+      const selected = readRealtimeVoiceMicrophoneId();
+      void refreshMicrophones().then((inputs) => {
+        if (
+          inputs &&
+          selected !== "default" &&
+          !inputs.some((device) => device.deviceId === selected)
+        ) {
+          void setMicrophone("default");
+        }
+      });
+    };
+    mediaDevices.addEventListener("devicechange", handleDeviceChange);
+    return () =>
+      mediaDevices.removeEventListener("devicechange", handleDeviceChange);
+  }, [refreshMicrophones, setMicrophone]);
+
   useEffect(() => {
     const onSidebarState = (event: Event) => {
       const detail = (event as CustomEvent<AgentSidebarStateChangeDetail>)
@@ -848,6 +1911,16 @@ function useRealtimeVoiceModeController(
     errorMessage: error,
     chatVisible,
     audioLevels,
+    preferences,
+    microphones,
+    microphoneDeviceId,
+    microphoneSwitching,
+    microphoneError,
+    voiceChangePending,
+    setLanguage,
+    setIntelligence,
+    setVoice,
+    setMicrophone,
     start,
     end,
     toggleChat,
@@ -868,6 +1941,77 @@ export function RealtimeVoiceModeProvider({
   );
   const copy = useRealtimeVoiceModeCopy();
   const voice = useRealtimeVoiceModeController(resolvedBrowserTabId, copy);
+  const inlineSettings = useMemo(
+    () => ({
+      dialogLabel: copy.voiceSettings,
+      ...(voice.voiceChangePending
+        ? { appliesNextConversationNote: copy.settings.voiceChangePending }
+        : {}),
+      ...(voice.microphoneError
+        ? { microphoneError: voice.microphoneError }
+        : {}),
+      microphone: {
+        label: copy.settings.microphone,
+        value: voice.microphoneDeviceId,
+        disabled: voice.microphoneSwitching,
+        options: [
+          {
+            value: "default",
+            label: copy.settings.defaultMicrophone,
+          },
+          ...voice.microphones.map((microphone) => ({
+            value: microphone.deviceId,
+            label: microphone.label,
+          })),
+        ],
+        onValueChange: (value: string) => {
+          void voice.setMicrophone(value);
+        },
+      },
+      language: {
+        label: copy.settings.language,
+        value: voice.preferences.language,
+        options: REALTIME_VOICE_LANGUAGES.map((language) => ({
+          value: language,
+          label:
+            language === "auto"
+              ? copy.settings.autoLanguage
+              : copy.settings.languages[language],
+        })),
+        onValueChange: (value: string) => {
+          if (isOneOf(REALTIME_VOICE_LANGUAGES, value)) {
+            voice.setLanguage(value);
+          }
+        },
+      },
+      intelligence: {
+        label: copy.settings.intelligence,
+        value: voice.preferences.intelligence,
+        options: REALTIME_VOICE_INTELLIGENCE_LEVELS.map((intelligence) => ({
+          value: intelligence,
+          label: copy.settings.intelligenceLevels[intelligence],
+        })),
+        onValueChange: (value: string) => {
+          if (isOneOf(REALTIME_VOICE_INTELLIGENCE_LEVELS, value)) {
+            voice.setIntelligence(value);
+          }
+        },
+      },
+      voiceStyle: {
+        label: copy.settings.voiceStyle,
+        value: voice.preferences.voice,
+        options: REALTIME_VOICES.map((voiceName) => ({
+          value: voiceName,
+          label: `${voiceName[0]!.toUpperCase()}${voiceName.slice(1)}`,
+          description: copy.settings.voiceDescriptions[voiceName],
+        })),
+        onValueChange: (value: string) => {
+          if (isOneOf(REALTIME_VOICES, value)) voice.setVoice(value);
+        },
+      },
+    }),
+    [copy, voice],
+  );
 
   return (
     <RealtimeVoiceModeContext.Provider value={voice}>
@@ -881,12 +2025,7 @@ export function RealtimeVoiceModeProvider({
               audioLevels={voice.audioLevels}
               onToggleChat={voice.toggleChat}
               onEndVoiceMode={voice.end}
-              onOpenMicrophoneSettings={() =>
-                endRealtimeVoiceBeforeNavigation(
-                  voice.end,
-                  openMicrophoneSettings,
-                )
-              }
+              settings={inlineSettings}
               errorMessage={voice.errorMessage}
             />,
             document.body,