@agent-native/toolkit 0.8.3 → 0.9.1

package/src/composer/RealtimeVoiceMode.spec.tsx

@@ -14,7 +14,7 @@ import {
 
 const copy: RealtimeVoiceModeCopy = {
   entryButtonLabel: "Use microphone",
-  promptTitle: "Talk to your app",
+  promptTitle: "Use your voice",
   promptDescription:
     "Voice mode keeps listening while the agent navigates and takes actions.",
   setupTitle: "Set up voice mode",
@@ -23,8 +23,9 @@ const copy: RealtimeVoiceModeCopy = {
   connectBuilder: "Connect Builder.io",
   useOpenAiKey: "Add your own keys",
   startWithOpenAiKey: "Start with OpenAI key",
-  startVoiceMode: "Start voice mode",
-  keepDictating: "Keep dictating",
+  startVoiceMode: "Real-time voice",
+  keepDictating: "Dictate",
+  rememberPreference: "Remember my preference",
   showChat: "Show chat",
   hideChat: "Hide chat",
   endVoiceMode: "End voice mode",
@@ -127,23 +128,27 @@ describe("RealtimeVoiceMode", () => {
 
     act(() => microphone?.click());
 
-    expect(document.body.textContent).toContain("Talk to your app");
+    expect(document.body.textContent).toContain("Use your voice");
     const prompt = document.querySelector<HTMLElement>('[role="dialog"]');
     expect(prompt?.className).toContain(
-      "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,22rem),22rem)]",
+      "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,18rem),18rem)]",
     );
-    expect(document.body.textContent).toContain("Keep dictating");
+    expect(document.body.textContent).not.toContain(
+      "Voice mode keeps listening while the agent navigates and takes actions.",
+    );
+    expect(document.body.textContent).toContain("Dictate");
+    expect(document.body.textContent).toContain("Remember my preference");
     expect(onStartVoiceMode).not.toHaveBeenCalled();
     expect(onKeepDictating).not.toHaveBeenCalled();
 
     const startVoiceMode = Array.from(
       document.querySelectorAll<HTMLButtonElement>("button"),
-    ).find((button) => button.textContent?.includes("Start voice mode"));
+    ).find((button) => button.textContent?.includes("Real-time voice"));
     act(() => startVoiceMode?.click());
 
     expect(onStartVoiceMode).toHaveBeenCalledOnce();
     expect(onKeepDictating).not.toHaveBeenCalled();
-    expect(document.body.textContent).not.toContain("Talk to your app");
+    expect(document.body.textContent).not.toContain("Use your voice");
   });
 
   it("does not start realtime voice while provider readiness is unresolved", () => {
@@ -162,14 +167,14 @@ describe("RealtimeVoiceMode", () => {
 
     const startVoiceMode = Array.from(
       document.querySelectorAll<HTMLButtonElement>("button"),
-    ).find((button) => button.textContent?.includes("Start voice mode"));
+    ).find((button) => button.textContent?.includes("Real-time voice"));
     expect(startVoiceMode?.disabled).toBe(true);
     act(() => startVoiceMode?.click());
     expect(onStartVoiceMode).not.toHaveBeenCalled();
 
     const keepDictating = Array.from(
       document.querySelectorAll<HTMLButtonElement>("button"),
-    ).find((button) => button.textContent === "Keep dictating");
+    ).find((button) => button.textContent === "Dictate");
     expect(keepDictating?.disabled).toBe(false);
   });
 
@@ -188,13 +193,83 @@ describe("RealtimeVoiceMode", () => {
 
     const keepDictating = Array.from(
       document.querySelectorAll<HTMLButtonElement>("button"),
-    ).find((button) => button.textContent === "Keep dictating");
+    ).find((button) => button.textContent === "Dictate");
     act(() => keepDictating?.click());
 
     expect(onKeepDictating).toHaveBeenCalledOnce();
     expect(onStartVoiceMode).not.toHaveBeenCalled();
   });
 
+  it("remembers the selected input mode only when requested", () => {
+    const onStartVoiceMode = vi.fn();
+    const onRememberPreference = vi.fn();
+
+    render(
+      <RealtimeVoiceModeEntry
+        copy={copy}
+        open
+        onStartVoiceMode={onStartVoiceMode}
+        onKeepDictating={vi.fn()}
+        onRememberPreference={onRememberPreference}
+      />,
+    );
+
+    const remember = document.querySelector<HTMLElement>('[role="checkbox"]');
+    act(() => remember?.click());
+    const realtime = Array.from(
+      document.querySelectorAll<HTMLButtonElement>("button"),
+    ).find((button) => button.textContent?.includes("Real-time voice"));
+    act(() => realtime?.click());
+
+    expect(onRememberPreference).toHaveBeenCalledWith("realtime");
+    expect(onStartVoiceMode).toHaveBeenCalledOnce();
+  });
+
+  it("bypasses the chooser for a remembered input mode", () => {
+    const onStartVoiceMode = vi.fn();
+    const onKeepDictating = vi.fn();
+
+    render(
+      <RealtimeVoiceModeEntry
+        copy={copy}
+        preferredMode="dictation"
+        onStartVoiceMode={onStartVoiceMode}
+        onKeepDictating={onKeepDictating}
+      />,
+    );
+
+    const microphone = document.querySelector<HTMLButtonElement>(
+      'button[aria-label="Use microphone"]',
+    );
+    act(() => microphone?.click());
+
+    expect(onKeepDictating).toHaveBeenCalledOnce();
+    expect(onStartVoiceMode).not.toHaveBeenCalled();
+    expect(document.body.textContent).not.toContain("Use your voice");
+  });
+
+  it("shows setup instead of starting a remembered realtime mode without a provider", () => {
+    const onStartVoiceMode = vi.fn();
+
+    render(
+      <RealtimeVoiceModeEntry
+        copy={copy}
+        setupRequired
+        preferredMode="realtime"
+        onStartVoiceMode={onStartVoiceMode}
+        onKeepDictating={vi.fn()}
+      />,
+    );
+
+    const microphone = document.querySelector<HTMLButtonElement>(
+      'button[aria-label="Use microphone"]',
+    );
+    act(() => microphone?.click());
+
+    expect(onStartVoiceMode).not.toHaveBeenCalled();
+    expect(document.body.textContent).toContain("Set up voice mode");
+  });
+
   it("makes Builder the primary setup action and own keys the secondary", () => {
     const onConnectBuilder = vi.fn();
     const onUseOpenAiKey = vi.fn();
@@ -653,6 +728,10 @@ describe("RealtimeVoiceMode", () => {
     expect(
       document.querySelector('[data-realtime-voice-waveform="true"]'),
     ).not.toBeNull();
+    const centerBar = document.querySelectorAll<HTMLElement>(
+      '[data-realtime-voice-waveform="true"] > span',
+    )[2];
+    expect(centerBar?.style.transform).toBe("scaleY(0.76)");
 
     act(() => audioLevels.set({ input: 0, output: 0.8 }));
     expect(
@@ -687,6 +766,32 @@ describe("RealtimeVoiceMode", () => {
     },
   );
 
+  it("shines only while the agent is working", () => {
+    render(
+      <RealtimeVoiceModeDock
+        state="working"
+        copy={copy}
+        chatVisible={false}
+        onToggleChat={vi.fn()}
+        onEndVoiceMode={vi.fn()}
+      />,
+    );
+    expect(
+      document.querySelector(".agent-realtime-voice-working"),
+    ).not.toBeNull();
+
+    render(
+      <RealtimeVoiceModeDock
+        state="listening"
+        copy={copy}
+        chatVisible={false}
+        onToggleChat={vi.fn()}
+        onEndVoiceMode={vi.fn()}
+      />,
+    );
+    expect(document.querySelector(".agent-realtime-voice-working")).toBeNull();
+  });
+
   it("shows an unmistakable loader and ignores early audio until connected", () => {
     const audioLevels = createRealtimeVoiceAudioLevelStore();
     audioLevels.set({ input: 0.8, output: 0 });

package/src/composer/RealtimeVoiceMode.tsx

@@ -1,4 +1,5 @@
 import { Button } from "@agent-native/toolkit/ui/button";
+import { Checkbox } from "@agent-native/toolkit/ui/checkbox";
 import {
   Select,
   SelectContent,
@@ -62,6 +63,7 @@ export interface RealtimeVoiceModeCopy {
   startWithOpenAiKey: string;
   startVoiceMode: string;
   keepDictating: string;
+  rememberPreference: string;
   showChat: string;
   hideChat: string;
   endVoiceMode: string;
@@ -111,6 +113,8 @@ export interface RealtimeVoiceModeEntryProps {
   onOpenChange?: (open: boolean) => void;
   onStartVoiceMode: () => void;
   onKeepDictating: () => void;
+  preferredMode?: RealtimeVoiceInputMode | null;
+  onRememberPreference?: (mode: RealtimeVoiceInputMode) => void;
   setupRequired?: boolean;
   openAiConfigured?: boolean;
   connectingBuilder?: boolean;
@@ -119,6 +123,8 @@ export interface RealtimeVoiceModeEntryProps {
   className?: string;
 }
 
+export type RealtimeVoiceInputMode = "realtime" | "dictation";
+
 /**
  * Composer mic entry point for apps that support a full-duplex voice session.
  * The first click offers voice mode without silently changing the existing
@@ -132,6 +138,8 @@ export function RealtimeVoiceModeEntry({
   onOpenChange,
   onStartVoiceMode,
   onKeepDictating,
+  preferredMode = null,
+  onRememberPreference,
   setupRequired = false,
   openAiConfigured = false,
   connectingBuilder = false,
@@ -143,6 +151,8 @@ export function RealtimeVoiceModeEntry({
   const open = controlledOpen ?? uncontrolledOpen;
   const titleId = useId();
   const descriptionId = useId();
+  const rememberPreferenceId = useId();
+  const [rememberPreference, setRememberPreference] = useState(false);
   const [collisionBoundary, setCollisionBoundary] =
     useState<HTMLElement | null>(null);
 
@@ -159,8 +169,9 @@ export function RealtimeVoiceModeEntry({
     onOpenChange?.(nextOpen);
   };
 
-  const choose = (callback: () => void) => {
+  const choose = (mode: RealtimeVoiceInputMode, callback: () => void) => {
     setOpen(false);
+    if (rememberPreference) onRememberPreference?.(mode);
     callback();
   };
 
@@ -177,6 +188,23 @@ export function RealtimeVoiceModeEntry({
               disabled={disabled}
               aria-label={copy.entryButtonLabel}
               aria-expanded={open}
+              onClick={(event) => {
+                if (!preferredMode) return;
+                if (
+                  preferredMode === "realtime" &&
+                  (providerStatusPending || setupRequired)
+                ) {
+                  return;
+                }
+                event.preventDefault();
+                event.stopPropagation();
+                choose(
+                  preferredMode,
+                  preferredMode === "realtime"
+                    ? onStartVoiceMode
+                    : onKeepDictating,
+                );
+              }}
               className={cn(
                 "size-7 shrink-0 text-muted-foreground hover:text-foreground",
                 className,
@@ -197,43 +225,54 @@ export function RealtimeVoiceModeEntry({
         collisionPadding={16}
         data-collision-boundary={collisionBoundary ? "agent-panel" : "viewport"}
         className={cn(
-          "p-4",
+          setupRequired ? "p-4" : "p-3",
           setupRequired
             ? "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,30rem),30rem)]"
-            : "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,22rem),22rem)]",
+            : "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,18rem),18rem)]",
         )}
         aria-labelledby={titleId}
-        aria-describedby={descriptionId}
+        aria-describedby={setupRequired ? descriptionId : undefined}
       >
-        <div className="grid gap-3">
+        <div className={cn("grid", setupRequired ? "gap-3" : "gap-2.5")}>
           <div className="grid gap-1">
             <h2 id={titleId} className="text-sm font-semibold text-foreground">
               {setupRequired ? copy.setupTitle : copy.promptTitle}
             </h2>
-            <p
-              id={descriptionId}
-              className="text-sm leading-relaxed text-muted-foreground"
-            >
-              {setupRequired ? copy.setupDescription : copy.promptDescription}
-            </p>
+            {setupRequired ? (
+              <p
+                id={descriptionId}
+                className="text-sm leading-relaxed text-muted-foreground"
+              >
+                {copy.setupDescription}
+              </p>
+            ) : null}
           </div>
 
-          <div className="flex flex-col-reverse gap-2 sm:flex-row sm:flex-wrap sm:justify-end">
-            <Button
-              type="button"
-              variant="ghost"
-              size="sm"
-              onClick={() => choose(onKeepDictating)}
-            >
-              {copy.keepDictating}
-            </Button>
+          <div
+            className={cn(
+              "gap-2",
+              setupRequired
+                ? "flex flex-col-reverse sm:flex-row sm:flex-wrap sm:justify-end"
+                : "grid",
+            )}
+          >
             {setupRequired ? (
               <>
+                <Button
+                  type="button"
+                  variant="ghost"
+                  size="sm"
+                  onClick={() => choose("dictation", onKeepDictating)}
+                >
+                  {copy.keepDictating}
+                </Button>
                 <Button
                   type="button"
                   variant="outline"
                   size="sm"
-                  onClick={() => choose(onUseOpenAiKey ?? onStartVoiceMode)}
+                  onClick={() =>
+                    choose("realtime", onUseOpenAiKey ?? onStartVoiceMode)
+                  }
                 >
                   {openAiConfigured
                     ? copy.startWithOpenAiKey
@@ -244,7 +283,9 @@ export function RealtimeVoiceModeEntry({
                   size="sm"
                   className="whitespace-nowrap"
                   disabled={connectingBuilder}
-                  onClick={() => choose(onConnectBuilder ?? onStartVoiceMode)}
+                  onClick={() =>
+                    choose("realtime", onConnectBuilder ?? onStartVoiceMode)
+                  }
                 >
                   {connectingBuilder ? (
                     <IconLoader2 className="animate-spin" />
@@ -253,21 +294,49 @@ export function RealtimeVoiceModeEntry({
                 </Button>
               </>
             ) : (
-              <Button
-                type="button"
-                size="sm"
-                disabled={providerStatusPending}
-                onClick={() => choose(onStartVoiceMode)}
-              >
-                {providerStatusPending ? (
-                  <IconLoader2 className="animate-spin" />
-                ) : (
-                  <IconMicrophone />
-                )}
-                {copy.startVoiceMode}
-              </Button>
+              <>
+                <Button
+                  type="button"
+                  size="sm"
+                  className="w-full"
+                  disabled={providerStatusPending}
+                  onClick={() => choose("realtime", onStartVoiceMode)}
+                >
+                  {providerStatusPending ? (
+                    <IconLoader2 className="animate-spin" />
+                  ) : (
+                    <IconMicrophone />
+                  )}
+                  {copy.startVoiceMode}
+                </Button>
+                <Button
+                  type="button"
+                  variant="outline"
+                  size="sm"
+                  className="w-full"
+                  onClick={() => choose("dictation", onKeepDictating)}
+                >
+                  {copy.keepDictating}
+                </Button>
+              </>
             )}
           </div>
+
+          {!setupRequired ? (
+            <label
+              htmlFor={rememberPreferenceId}
+              className="flex cursor-pointer items-center gap-2 text-xs text-muted-foreground"
+            >
+              <Checkbox
+                id={rememberPreferenceId}
+                checked={rememberPreference}
+                onCheckedChange={(checked) =>
+                  setRememberPreference(checked === true)
+                }
+              />
+              {copy.rememberPreference}
+            </label>
+          ) : null}
         </div>
       </PopoverContent>
     </Popover>
@@ -312,7 +381,7 @@ export interface RealtimeVoiceModeInlineSettings {
 
 const SILENT_AUDIO_LEVELS = createRealtimeVoiceAudioLevelStore();
 const WAVEFORM_WEIGHTS = [0.55, 0.82, 1, 0.82, 0.55];
-const AUDIO_ACTIVITY_THRESHOLD = 0.035;
+const AUDIO_ACTIVITY_THRESHOLD = 0.1;
 
 function usePrefersReducedMotion(): boolean {
   const [reduced, setReduced] = useState(false);
@@ -831,6 +900,7 @@ export function RealtimeVoiceModeDock({
               className={cn(
                 "relative isolate size-16 overflow-visible rounded-full ring-1 backdrop-blur-xl transition-transform duration-150 ease-out focus-visible:ring-offset-2 active:scale-[0.97] motion-reduce:transition-none",
                 ORB_STATE_CLASSES[state],
+                state === "working" && "agent-realtime-voice-working",
               )}
             >
               <span className="relative z-10 flex items-center justify-center">

package/src/composer/VoiceButton.spec.ts

@@ -1,6 +1,22 @@
 import { describe, expect, it } from "vitest";
 
-import { isRealtimeVoiceSetupRequired } from "./VoiceButton.js";
+import {
+  isRealtimeVoiceSetupRequired,
+  normalizeVoiceInputPreference,
+} from "./VoiceButton.js";
+
+describe("normalizeVoiceInputPreference", () => {
+  it("accepts only persisted realtime and dictation choices", () => {
+    expect(normalizeVoiceInputPreference({ mode: "realtime" })).toBe(
+      "realtime",
+    );
+    expect(normalizeVoiceInputPreference({ mode: "dictation" })).toBe(
+      "dictation",
+    );
+    expect(normalizeVoiceInputPreference({ mode: "voice" })).toBeNull();
+    expect(normalizeVoiceInputPreference(null)).toBeNull();
+  });
+});
 
 describe("isRealtimeVoiceSetupRequired", () => {
   it("waits for the voice-specific provider status before prompting setup", () => {

package/src/composer/VoiceButton.tsx

@@ -14,9 +14,11 @@ import {
   IconLoader2,
   IconX,
 } from "@tabler/icons-react";
+import { useCallback, useEffect, useState } from "react";
 
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip.js";
 import { RealtimeVoiceModeEntry } from "./RealtimeVoiceMode.js";
+import type { RealtimeVoiceInputMode } from "./RealtimeVoiceMode.js";
 import { useComposerRuntimeAdapters } from "./runtime-adapters.js";
 import {
   useRealtimeVoiceModeCopy,
@@ -24,6 +26,16 @@ import {
 } from "./useRealtimeVoiceMode.js";
 import type { VoiceDictationApi } from "./useVoiceDictation.js";
 
+const VOICE_INPUT_PREFERENCE_KEY = "voice-input-preference";
+
+export function normalizeVoiceInputPreference(
+  value: unknown,
+): RealtimeVoiceInputMode | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const mode = (value as { mode?: unknown }).mode;
+  return mode === "realtime" || mode === "dictation" ? mode : null;
+}
+
 function openOpenAiKeySettings(): void {
   window.location.hash = "#secrets:OPENAI_API_KEY";
   window.dispatchEvent(new Event("agent-panel:open"));
@@ -63,6 +75,41 @@ export function VoiceButton({ voice, isMac, disabled }: VoiceButtonProps) {
     trackingFlow: "voice_transcription",
     onConnected: () => voiceProviders.refresh(),
   });
+  const [inputPreference, setInputPreference] = useState<{
+    ready: boolean;
+    mode: RealtimeVoiceInputMode | null;
+  }>({ ready: false, mode: null });
+
+  useEffect(() => {
+    let cancelled = false;
+    void Promise.resolve(
+      adapters.voice!.readAppState!(VOICE_INPUT_PREFERENCE_KEY),
+    )
+      .then((value) => {
+        if (!cancelled) {
+          setInputPreference({
+            ready: true,
+            mode: normalizeVoiceInputPreference(value),
+          });
+        }
+      })
+      .catch(() => {
+        if (!cancelled) setInputPreference({ ready: true, mode: null });
+      });
+    return () => {
+      cancelled = true;
+    };
+  }, [adapters]);
+
+  const rememberInputPreference = useCallback(
+    (mode: RealtimeVoiceInputMode) => {
+      setInputPreference({ ready: true, mode });
+      void Promise.resolve(
+        adapters.voice!.setAppState!(VOICE_INPUT_PREFERENCE_KEY, { mode }),
+      ).catch(() => undefined);
+    },
+    [adapters],
+  );
 
   if (!supported) return null;
 
@@ -90,6 +137,8 @@ export function VoiceButton({ voice, isMac, disabled }: VoiceButtonProps) {
         }}
         onStartVoiceMode={() => void realtimeVoice.start()}
         onKeepDictating={() => void start()}
+        preferredMode={inputPreference.ready ? inputPreference.mode : null}
+        onRememberPreference={rememberInputPreference}
       />
     );
   }

package/src/composer/realtime-voice-audio-level.spec.ts

@@ -12,6 +12,21 @@ describe("realtime voice audio levels", () => {
     expect(normalizeRealtimeVoiceRms(new Uint8Array(32).fill(255))).toBe(1);
   });
 
+  it("makes ordinary speech visibly responsive above the noise floor", () => {
+    const quietSpeech = Uint8Array.from({ length: 32 }, (_, index) =>
+      index % 2 === 0 ? 124 : 132,
+    );
+    const typicalSpeech = Uint8Array.from({ length: 32 }, (_, index) =>
+      index % 2 === 0 ? 120 : 136,
+    );
+
+    expect(normalizeRealtimeVoiceRms(quietSpeech)).toBeGreaterThan(0.3);
+    expect(normalizeRealtimeVoiceRms(typicalSpeech)).toBeGreaterThan(0.5);
+    expect(normalizeRealtimeVoiceRms(typicalSpeech)).toBeGreaterThan(
+      normalizeRealtimeVoiceRms(quietSpeech),
+    );
+  });
+
   it("attacks faster than it decays", () => {
     expect(smoothRealtimeVoiceLevel(0, 1)).toBe(0.55);
     expect(smoothRealtimeVoiceLevel(1, 0)).toBe(0.8);

package/src/composer/realtime-voice-audio-level.ts

@@ -25,7 +25,8 @@ export function normalizeRealtimeVoiceRms(samples: Uint8Array): number {
     sumSquares += normalized * normalized;
   }
   const rms = Math.sqrt(sumSquares / samples.length);
-  return clampLevel(Math.max(0, rms - 0.012) * 3.2);
+  const activeLevel = clampLevel((rms - 0.012) / 0.18);
+  return Math.sqrt(activeLevel);
 }
 
 export function smoothRealtimeVoiceLevel(

package/src/composer/useRealtimeVoiceMode.tsx

@@ -1018,6 +1018,7 @@ function voiceCopy(t: (key: string) => string): RealtimeVoiceModeCopy {
     startWithOpenAiKey: t("agentPanel.voiceMode.startWithOpenAiKey"),
     startVoiceMode: t("agentPanel.voiceMode.start"),
     keepDictating: t("agentPanel.voiceMode.keepDictating"),
+    rememberPreference: t("agentPanel.voiceMode.rememberPreference"),
     showChat: t("agentPanel.voiceMode.showChat"),
     hideChat: t("agentPanel.voiceMode.hideChat"),
     endVoiceMode: t("agentPanel.voiceMode.end"),
@@ -1353,7 +1354,7 @@ function useRealtimeVoiceModeController(
         analyser.fftSize = 512;
         const source = context.createMediaStreamSource(stream);
         source.connect(analyser);
-        const buffer = new Uint8Array(analyser.frequencyBinCount);
+        const buffer = new Uint8Array(analyser.fftSize);
         if (channel === "input") {
           inputSourceRef.current?.disconnect();
           inputSourceRef.current = source;
@@ -1728,9 +1729,10 @@ function useRealtimeVoiceModeController(
       audio.setAttribute("playsinline", "");
       audioRef.current = audio;
       peer.ontrack = (trackEvent) => {
-        const remoteStream = trackEvent.streams[0] ?? null;
+        const remoteStream =
+          trackEvent.streams[0] ?? new MediaStream([trackEvent.track]);
         audio.srcObject = remoteStream;
-        if (remoteStream) attachAudioMeter(remoteStream, "output");
+        attachAudioMeter(remoteStream, "output");
         void audio.play().catch(() => undefined);
       };