@agent-native/toolkit 0.9.0 → 0.10.0

package/src/composer/RealtimeVoiceMode.tsx

@@ -1,4 +1,5 @@
 import { Button } from "@agent-native/toolkit/ui/button";
+import { Checkbox } from "@agent-native/toolkit/ui/checkbox";
 import {
   Select,
   SelectContent,
@@ -62,6 +63,7 @@ export interface RealtimeVoiceModeCopy {
   startWithOpenAiKey: string;
   startVoiceMode: string;
   keepDictating: string;
+  rememberPreference: string;
   showChat: string;
   hideChat: string;
   endVoiceMode: string;
@@ -111,6 +113,8 @@ export interface RealtimeVoiceModeEntryProps {
   onOpenChange?: (open: boolean) => void;
   onStartVoiceMode: () => void;
   onKeepDictating: () => void;
+  preferredMode?: RealtimeVoiceInputMode | null;
+  onRememberPreference?: (mode: RealtimeVoiceInputMode) => void;
   setupRequired?: boolean;
   openAiConfigured?: boolean;
   connectingBuilder?: boolean;
@@ -119,6 +123,8 @@ export interface RealtimeVoiceModeEntryProps {
   className?: string;
 }
 
+export type RealtimeVoiceInputMode = "realtime" | "dictation";
+
 /**
  * Composer mic entry point for apps that support a full-duplex voice session.
  * The first click offers voice mode without silently changing the existing
@@ -132,6 +138,8 @@ export function RealtimeVoiceModeEntry({
   onOpenChange,
   onStartVoiceMode,
   onKeepDictating,
+  preferredMode = null,
+  onRememberPreference,
   setupRequired = false,
   openAiConfigured = false,
   connectingBuilder = false,
@@ -143,6 +151,8 @@ export function RealtimeVoiceModeEntry({
   const open = controlledOpen ?? uncontrolledOpen;
   const titleId = useId();
   const descriptionId = useId();
+  const rememberPreferenceId = useId();
+  const [rememberPreference, setRememberPreference] = useState(false);
   const [collisionBoundary, setCollisionBoundary] =
     useState<HTMLElement | null>(null);
 
@@ -159,8 +169,9 @@ export function RealtimeVoiceModeEntry({
     onOpenChange?.(nextOpen);
   };
 
-  const choose = (callback: () => void) => {
+  const choose = (mode: RealtimeVoiceInputMode, callback: () => void) => {
     setOpen(false);
+    if (rememberPreference) onRememberPreference?.(mode);
     callback();
   };
 
@@ -177,6 +188,23 @@ export function RealtimeVoiceModeEntry({
               disabled={disabled}
               aria-label={copy.entryButtonLabel}
               aria-expanded={open}
+              onClick={(event) => {
+                if (!preferredMode) return;
+                if (
+                  preferredMode === "realtime" &&
+                  (providerStatusPending || setupRequired)
+                ) {
+                  return;
+                }
+                event.preventDefault();
+                event.stopPropagation();
+                choose(
+                  preferredMode,
+                  preferredMode === "realtime"
+                    ? onStartVoiceMode
+                    : onKeepDictating,
+                );
+              }}
               className={cn(
                 "size-7 shrink-0 text-muted-foreground hover:text-foreground",
                 className,
@@ -197,43 +225,54 @@ export function RealtimeVoiceModeEntry({
         collisionPadding={16}
         data-collision-boundary={collisionBoundary ? "agent-panel" : "viewport"}
         className={cn(
-          "p-4",
+          setupRequired ? "p-4" : "p-3",
           setupRequired
             ? "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,30rem),30rem)]"
-            : "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,22rem),22rem)]",
+            : "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,18rem),18rem)]",
         )}
         aria-labelledby={titleId}
-        aria-describedby={descriptionId}
+        aria-describedby={setupRequired ? descriptionId : undefined}
       >
-        <div className="grid gap-3">
+        <div className={cn("grid", setupRequired ? "gap-3" : "gap-2.5")}>
           <div className="grid gap-1">
             <h2 id={titleId} className="text-sm font-semibold text-foreground">
               {setupRequired ? copy.setupTitle : copy.promptTitle}
             </h2>
-            <p
-              id={descriptionId}
-              className="text-sm leading-relaxed text-muted-foreground"
-            >
-              {setupRequired ? copy.setupDescription : copy.promptDescription}
-            </p>
+            {setupRequired ? (
+              <p
+                id={descriptionId}
+                className="text-sm leading-relaxed text-muted-foreground"
+              >
+                {copy.setupDescription}
+              </p>
+            ) : null}
           </div>
 
-          <div className="flex flex-col-reverse gap-2 sm:flex-row sm:flex-wrap sm:justify-end">
-            <Button
-              type="button"
-              variant="ghost"
-              size="sm"
-              onClick={() => choose(onKeepDictating)}
-            >
-              {copy.keepDictating}
-            </Button>
+          <div
+            className={cn(
+              "gap-2",
+              setupRequired
+                ? "flex flex-col-reverse sm:flex-row sm:flex-wrap sm:justify-end"
+                : "grid",
+            )}
+          >
             {setupRequired ? (
               <>
+                <Button
+                  type="button"
+                  variant="ghost"
+                  size="sm"
+                  onClick={() => choose("dictation", onKeepDictating)}
+                >
+                  {copy.keepDictating}
+                </Button>
                 <Button
                   type="button"
                   variant="outline"
                   size="sm"
-                  onClick={() => choose(onUseOpenAiKey ?? onStartVoiceMode)}
+                  onClick={() =>
+                    choose("realtime", onUseOpenAiKey ?? onStartVoiceMode)
+                  }
                 >
                   {openAiConfigured
                     ? copy.startWithOpenAiKey
@@ -244,7 +283,9 @@ export function RealtimeVoiceModeEntry({
                   size="sm"
                   className="whitespace-nowrap"
                   disabled={connectingBuilder}
-                  onClick={() => choose(onConnectBuilder ?? onStartVoiceMode)}
+                  onClick={() =>
+                    choose("realtime", onConnectBuilder ?? onStartVoiceMode)
+                  }
                 >
                   {connectingBuilder ? (
                     <IconLoader2 className="animate-spin" />
@@ -253,21 +294,49 @@ export function RealtimeVoiceModeEntry({
                 </Button>
               </>
             ) : (
-              <Button
-                type="button"
-                size="sm"
-                disabled={providerStatusPending}
-                onClick={() => choose(onStartVoiceMode)}
-              >
-                {providerStatusPending ? (
-                  <IconLoader2 className="animate-spin" />
-                ) : (
-                  <IconMicrophone />
-                )}
-                {copy.startVoiceMode}
-              </Button>
+              <>
+                <Button
+                  type="button"
+                  size="sm"
+                  className="w-full"
+                  disabled={providerStatusPending}
+                  onClick={() => choose("realtime", onStartVoiceMode)}
+                >
+                  {providerStatusPending ? (
+                    <IconLoader2 className="animate-spin" />
+                  ) : (
+                    <IconMicrophone />
+                  )}
+                  {copy.startVoiceMode}
+                </Button>
+                <Button
+                  type="button"
+                  variant="outline"
+                  size="sm"
+                  className="w-full"
+                  onClick={() => choose("dictation", onKeepDictating)}
+                >
+                  {copy.keepDictating}
+                </Button>
+              </>
             )}
           </div>
+
+          {!setupRequired ? (
+            <label
+              htmlFor={rememberPreferenceId}
+              className="flex cursor-pointer items-center gap-2 text-xs text-muted-foreground"
+            >
+              <Checkbox
+                id={rememberPreferenceId}
+                checked={rememberPreference}
+                onCheckedChange={(checked) =>
+                  setRememberPreference(checked === true)
+                }
+              />
+              {copy.rememberPreference}
+            </label>
+          ) : null}
         </div>
       </PopoverContent>
     </Popover>
@@ -312,7 +381,7 @@ export interface RealtimeVoiceModeInlineSettings {
 
 const SILENT_AUDIO_LEVELS = createRealtimeVoiceAudioLevelStore();
 const WAVEFORM_WEIGHTS = [0.55, 0.82, 1, 0.82, 0.55];
-const AUDIO_ACTIVITY_THRESHOLD = 0.035;
+const AUDIO_ACTIVITY_THRESHOLD = 0.1;
 
 function usePrefersReducedMotion(): boolean {
   const [reduced, setReduced] = useState(false);
@@ -831,6 +900,7 @@ export function RealtimeVoiceModeDock({
               className={cn(
                 "relative isolate size-16 overflow-visible rounded-full ring-1 backdrop-blur-xl transition-transform duration-150 ease-out focus-visible:ring-offset-2 active:scale-[0.97] motion-reduce:transition-none",
                 ORB_STATE_CLASSES[state],
+                state === "working" && "agent-realtime-voice-working",
               )}
             >
               <span className="relative z-10 flex items-center justify-center">

package/src/composer/VoiceButton.spec.ts

@@ -1,6 +1,22 @@
 import { describe, expect, it } from "vitest";
 
-import { isRealtimeVoiceSetupRequired } from "./VoiceButton.js";
+import {
+  isRealtimeVoiceSetupRequired,
+  normalizeVoiceInputPreference,
+} from "./VoiceButton.js";
+
+describe("normalizeVoiceInputPreference", () => {
+  it("accepts only persisted realtime and dictation choices", () => {
+    expect(normalizeVoiceInputPreference({ mode: "realtime" })).toBe(
+      "realtime",
+    );
+    expect(normalizeVoiceInputPreference({ mode: "dictation" })).toBe(
+      "dictation",
+    );
+    expect(normalizeVoiceInputPreference({ mode: "voice" })).toBeNull();
+    expect(normalizeVoiceInputPreference(null)).toBeNull();
+  });
+});
 
 describe("isRealtimeVoiceSetupRequired", () => {
   it("waits for the voice-specific provider status before prompting setup", () => {

package/src/composer/VoiceButton.tsx

@@ -14,9 +14,11 @@ import {
   IconLoader2,
   IconX,
 } from "@tabler/icons-react";
+import { useCallback, useEffect, useState } from "react";
 
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip.js";
 import { RealtimeVoiceModeEntry } from "./RealtimeVoiceMode.js";
+import type { RealtimeVoiceInputMode } from "./RealtimeVoiceMode.js";
 import { useComposerRuntimeAdapters } from "./runtime-adapters.js";
 import {
   useRealtimeVoiceModeCopy,
@@ -24,6 +26,16 @@ import {
 } from "./useRealtimeVoiceMode.js";
 import type { VoiceDictationApi } from "./useVoiceDictation.js";
 
+const VOICE_INPUT_PREFERENCE_KEY = "voice-input-preference";
+
+export function normalizeVoiceInputPreference(
+  value: unknown,
+): RealtimeVoiceInputMode | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const mode = (value as { mode?: unknown }).mode;
+  return mode === "realtime" || mode === "dictation" ? mode : null;
+}
+
 function openOpenAiKeySettings(): void {
   window.location.hash = "#secrets:OPENAI_API_KEY";
   window.dispatchEvent(new Event("agent-panel:open"));
@@ -63,6 +75,41 @@ export function VoiceButton({ voice, isMac, disabled }: VoiceButtonProps) {
     trackingFlow: "voice_transcription",
     onConnected: () => voiceProviders.refresh(),
   });
+  const [inputPreference, setInputPreference] = useState<{
+    ready: boolean;
+    mode: RealtimeVoiceInputMode | null;
+  }>({ ready: false, mode: null });
+
+  useEffect(() => {
+    let cancelled = false;
+    void Promise.resolve(
+      adapters.voice!.readAppState!(VOICE_INPUT_PREFERENCE_KEY),
+    )
+      .then((value) => {
+        if (!cancelled) {
+          setInputPreference({
+            ready: true,
+            mode: normalizeVoiceInputPreference(value),
+          });
+        }
+      })
+      .catch(() => {
+        if (!cancelled) setInputPreference({ ready: true, mode: null });
+      });
+    return () => {
+      cancelled = true;
+    };
+  }, [adapters]);
+
+  const rememberInputPreference = useCallback(
+    (mode: RealtimeVoiceInputMode) => {
+      setInputPreference({ ready: true, mode });
+      void Promise.resolve(
+        adapters.voice!.setAppState!(VOICE_INPUT_PREFERENCE_KEY, { mode }),
+      ).catch(() => undefined);
+    },
+    [adapters],
+  );
 
   if (!supported) return null;
 
@@ -90,6 +137,8 @@ export function VoiceButton({ voice, isMac, disabled }: VoiceButtonProps) {
         }}
         onStartVoiceMode={() => void realtimeVoice.start()}
         onKeepDictating={() => void start()}
+        preferredMode={inputPreference.ready ? inputPreference.mode : null}
+        onRememberPreference={rememberInputPreference}
       />
     );
   }

package/src/composer/realtime-voice-audio-level.spec.ts

@@ -12,6 +12,21 @@ describe("realtime voice audio levels", () => {
     expect(normalizeRealtimeVoiceRms(new Uint8Array(32).fill(255))).toBe(1);
   });
 
+  it("makes ordinary speech visibly responsive above the noise floor", () => {
+    const quietSpeech = Uint8Array.from({ length: 32 }, (_, index) =>
+      index % 2 === 0 ? 124 : 132,
+    );
+    const typicalSpeech = Uint8Array.from({ length: 32 }, (_, index) =>
+      index % 2 === 0 ? 120 : 136,
+    );
+
+    expect(normalizeRealtimeVoiceRms(quietSpeech)).toBeGreaterThan(0.3);
+    expect(normalizeRealtimeVoiceRms(typicalSpeech)).toBeGreaterThan(0.5);
+    expect(normalizeRealtimeVoiceRms(typicalSpeech)).toBeGreaterThan(
+      normalizeRealtimeVoiceRms(quietSpeech),
+    );
+  });
+
   it("attacks faster than it decays", () => {
     expect(smoothRealtimeVoiceLevel(0, 1)).toBe(0.55);
     expect(smoothRealtimeVoiceLevel(1, 0)).toBe(0.8);

package/src/composer/realtime-voice-audio-level.ts

@@ -25,7 +25,8 @@ export function normalizeRealtimeVoiceRms(samples: Uint8Array): number {
     sumSquares += normalized * normalized;
   }
   const rms = Math.sqrt(sumSquares / samples.length);
-  return clampLevel(Math.max(0, rms - 0.012) * 3.2);
+  const activeLevel = clampLevel((rms - 0.012) / 0.18);
+  return Math.sqrt(activeLevel);
 }
 
 export function smoothRealtimeVoiceLevel(

package/src/composer/useRealtimeVoiceMode.spec.ts

@@ -6,6 +6,7 @@ import {
   createRealtimeVoiceGreetingEvent,
   createRealtimeVoiceGreetingStarter,
   createRealtimeVoicePreferenceUpdate,
+  createRealtimeVoiceResponseCoordinator,
   createRealtimeVoiceSession,
   createRealtimeVoiceSessionWithCapability,
   createRealtimeVoiceTranscriptSequencer,
@@ -521,7 +522,7 @@ describe("Realtime voice dynamic tool manifests", () => {
 });
 
 describe("extractRealtimeVoiceFunctionCalls", () => {
-  it("uses the low-latency completed-arguments event", () => {
+  it("waits for the terminal response event before returning a function call", () => {
     expect(
       extractRealtimeVoiceFunctionCalls({
         type: "response.function_call_arguments.done",
@@ -529,13 +530,7 @@ describe("extractRealtimeVoiceFunctionCalls", () => {
         call_id: "call-1",
         arguments: '{"path":"/inbox"}',
       }),
-    ).toEqual([
-      {
-        name: "navigate",
-        callId: "call-1",
-        argumentsText: '{"path":"/inbox"}',
-      },
-    ]);
+    ).toEqual([]);
   });
 
   it("falls back to completed function items on response.done", () => {
@@ -543,6 +538,7 @@ describe("extractRealtimeVoiceFunctionCalls", () => {
       extractRealtimeVoiceFunctionCalls({
         type: "response.done",
         response: {
+          status: "completed",
           output: [
             { type: "message", role: "assistant" },
             {
@@ -562,6 +558,29 @@ describe("extractRealtimeVoiceFunctionCalls", () => {
       },
     ]);
   });
+
+  it("ignores function items from failed or cancelled responses", () => {
+    const output = [
+      {
+        type: "function_call",
+        name: "navigate",
+        call_id: "call-3",
+        arguments: "{}",
+      },
+    ];
+    expect(
+      extractRealtimeVoiceFunctionCalls({
+        type: "response.done",
+        response: { status: "failed", output },
+      }),
+    ).toEqual([]);
+    expect(
+      extractRealtimeVoiceFunctionCalls({
+        type: "response.done",
+        response: { status: "cancelled", output },
+      }),
+    ).toEqual([]);
+  });
 });
 
 describe("extractCompletedRealtimeVoiceTranscript", () => {
@@ -632,6 +651,61 @@ describe("Realtime voice startup and transcript ordering", () => {
     expect(send).toHaveBeenCalledTimes(2);
   });
 
+  it("queues response requests behind the active response and retries a rejected request", () => {
+    const sent: Record<string, unknown>[] = [];
+    const coordinator = createRealtimeVoiceResponseCoordinator((event) =>
+      sent.push(event),
+    );
+
+    coordinator.request();
+    coordinator.handleEvent({
+      type: "response.created",
+      response: { id: "resp-active" },
+    });
+    coordinator.request();
+    expect(sent).toHaveLength(1);
+
+    expect(
+      coordinator.handleError(
+        "Conversation already has an active response in progress: resp-active",
+      ),
+    ).toBe(true);
+    coordinator.handleEvent({
+      type: "response.done",
+      response: { id: "resp-active", status: "completed" },
+    });
+
+    expect(sent).toHaveLength(2);
+    expect(sent[1]).toEqual({ type: "response.create" });
+  });
+
+  it("drops queued work after a failed response so the next turn can start", () => {
+    const sent: Record<string, unknown>[] = [];
+    const coordinator = createRealtimeVoiceResponseCoordinator((event) =>
+      sent.push(event),
+    );
+
+    coordinator.request();
+    coordinator.handleEvent({ type: "response.created" });
+    coordinator.request();
+    coordinator.handleEvent({
+      type: "response.done",
+      response: { status: "failed" },
+    });
+
+    expect(sent).toHaveLength(1);
+    coordinator.request();
+    expect(sent).toHaveLength(2);
+  });
+
+  it("ignores unrelated realtime errors", () => {
+    const coordinator = createRealtimeVoiceResponseCoordinator(vi.fn());
+
+    expect(coordinator.handleError("The microphone was disconnected.")).toBe(
+      false,
+    );
+  });
+
   it("publishes in conversation order when user ASR finishes after the assistant", () => {
     const published: Array<{ role: string; text: string }> = [];
     const sequencer = createRealtimeVoiceTranscriptSequencer((transcript) => {