@agent-native/toolkit 0.13.2 → 0.13.4

package/src/composer/useVoiceDictation.browser.spec.tsx

@@ -0,0 +1,265 @@
+// @vitest-environment happy-dom
+
+import { act } from "react";
+import { createRoot } from "react-dom/client";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+  useVoiceDictation,
+  type VoiceDictationApi,
+} from "./useVoiceDictation.js";
+
+class FakeSpeechRecognition {
+  static last: FakeSpeechRecognition | undefined;
+  continuous = false;
+  interimResults = false;
+  lang = "";
+  onaudiostart: (() => void) | undefined;
+  onresult: ((event: unknown) => void) | undefined;
+  onerror: ((event: { error?: string }) => void) | undefined;
+  onend: (() => void) | undefined;
+  started = false;
+
+  constructor() {
+    FakeSpeechRecognition.last = this;
+  }
+
+  start() {
+    this.started = true;
+  }
+  stop() {}
+  abort() {}
+}
+
+class FakeMediaRecorder {
+  static last: FakeMediaRecorder | undefined;
+  static isTypeSupported = () => true;
+  ondataavailable: ((event: unknown) => void) | undefined;
+  onstop: (() => void) | undefined;
+  mimeType = "audio/webm";
+  started = false;
+
+  constructor() {
+    FakeMediaRecorder.last = this;
+  }
+
+  start() {
+    this.started = true;
+  }
+  stop() {}
+}
+
+function stubSpeechEnvironment(getUserMedia = vi.fn()) {
+  FakeSpeechRecognition.last = undefined;
+  FakeMediaRecorder.last = undefined;
+  vi.stubGlobal("SpeechRecognition", FakeSpeechRecognition);
+  vi.stubGlobal("webkitSpeechRecognition", FakeSpeechRecognition);
+  vi.stubGlobal("MediaRecorder", undefined);
+  Object.defineProperty(navigator, "mediaDevices", {
+    configurable: true,
+    value: { getUserMedia },
+  });
+  vi.stubGlobal(
+    "fetch",
+    vi.fn().mockResolvedValue(new Response("{}", { status: 404 })),
+  );
+  return getUserMedia;
+}
+
+async function renderVoiceDictation() {
+  const seen: VoiceDictationApi[] = [];
+  function Probe() {
+    seen.push(useVoiceDictation({ onTranscript: vi.fn() }));
+    return null;
+  }
+  const container = document.createElement("div");
+  document.body.append(container);
+  const root = createRoot(container);
+  await act(async () => {
+    root.render(<Probe />);
+  });
+  return {
+    latest: () => seen[seen.length - 1]!,
+    async cleanup() {
+      await act(async () => {
+        root.unmount();
+      });
+      container.remove();
+    },
+  };
+}
+
+beforeEach(() => {
+  (globalThis as Record<string, unknown>).IS_REACT_ACT_ENVIRONMENT = true;
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe("useVoiceDictation — browser speech path", () => {
+  it("reports a session that ends before the microphone ever opened", async () => {
+    stubSpeechEnvironment();
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    const recognition = FakeSpeechRecognition.last!;
+    expect(recognition.started).toBe(true);
+    await act(async () => {
+      recognition.onerror?.({ error: "aborted" });
+      recognition.onend?.();
+    });
+
+    expect(probe.latest().state).toBe("error");
+    expect(probe.latest().errorMessage).toContain(
+      "stopped before it captured any audio",
+    );
+    await probe.cleanup();
+  });
+
+  it("treats a silent but live session as an ordinary empty result", async () => {
+    stubSpeechEnvironment(vi.fn().mockRejectedValue(new Error("no device")));
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    const recognition = FakeSpeechRecognition.last!;
+    await act(async () => {
+      recognition.onaudiostart?.();
+      recognition.onerror?.({ error: "no-speech" });
+      recognition.onend?.();
+    });
+
+    expect(probe.latest().state).toBe("idle");
+    expect(probe.latest().errorMessage).toBeNull();
+    await probe.cleanup();
+  });
+
+  it("keeps a reported speech error visible when `end` follows `error`", async () => {
+    stubSpeechEnvironment(vi.fn().mockResolvedValue({ getTracks: () => [] }));
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    const recognition = FakeSpeechRecognition.last!;
+    await act(async () => {
+      recognition.onaudiostart?.();
+      recognition.onerror?.({ error: "network" });
+      recognition.onend?.();
+    });
+
+    expect(probe.latest().state).toBe("error");
+    expect(probe.latest().errorMessage).toContain("couldn't reach its service");
+    await probe.cleanup();
+  });
+
+  it("falls back to the upload path when the speech service is unreachable", async () => {
+    stubSpeechEnvironment(vi.fn().mockResolvedValue({ getTracks: () => [] }));
+    vi.stubGlobal("MediaRecorder", FakeMediaRecorder);
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    await act(async () => {
+      FakeSpeechRecognition.last!.onaudiostart?.();
+      FakeSpeechRecognition.last!.onerror?.({ error: "network" });
+      FakeSpeechRecognition.last!.onend?.();
+    });
+    await act(async () => {});
+
+    expect(FakeMediaRecorder.last?.started).toBe(true);
+    expect(probe.latest().state).toBe("recording");
+    expect(probe.latest().errorMessage).toBeNull();
+    await probe.cleanup();
+  });
+
+  it("delivers partial speech instead of failing over after a mid-session drop", async () => {
+    stubSpeechEnvironment(vi.fn().mockResolvedValue({ getTracks: () => [] }));
+    vi.stubGlobal("MediaRecorder", FakeMediaRecorder);
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    await act(async () => {
+      const recognition = FakeSpeechRecognition.last!;
+      recognition.onaudiostart?.();
+      recognition.onresult?.({
+        resultIndex: 0,
+        results: [{ 0: { transcript: "make it blue" }, isFinal: true }],
+      });
+      recognition.onerror?.({ error: "network" });
+      recognition.onend?.();
+    });
+    await act(async () => {});
+
+    expect(FakeMediaRecorder.last).toBeUndefined();
+    expect(probe.latest().state).toBe("idle");
+    await probe.cleanup();
+  });
+
+  it("falls back to the upload path when the recognizer has no speech backend", async () => {
+    const getUserMedia = stubSpeechEnvironment(
+      vi.fn().mockResolvedValue({ getTracks: () => [] }),
+    );
+    vi.stubGlobal("MediaRecorder", FakeMediaRecorder);
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    await act(async () => {
+      FakeSpeechRecognition.last!.onerror?.({ error: "aborted" });
+      FakeSpeechRecognition.last!.onend?.();
+    });
+    await act(async () => {});
+
+    expect(getUserMedia).toHaveBeenCalled();
+    expect(FakeMediaRecorder.last?.started).toBe(true);
+    expect(probe.latest().state).toBe("recording");
+    expect(probe.latest().errorMessage).toBeNull();
+    await probe.cleanup();
+  });
+
+  it("does not retry through the upload path after a denied microphone", async () => {
+    stubSpeechEnvironment(vi.fn().mockResolvedValue({ getTracks: () => [] }));
+    vi.stubGlobal("MediaRecorder", FakeMediaRecorder);
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    await act(async () => {
+      FakeSpeechRecognition.last!.onerror?.({ error: "not-allowed" });
+      FakeSpeechRecognition.last!.onend?.();
+    });
+    await act(async () => {});
+
+    expect(FakeMediaRecorder.last).toBeUndefined();
+    expect(probe.latest().state).toBe("error");
+    expect(probe.latest().errorMessage).toContain("site controls icon");
+    await probe.cleanup();
+  });
+
+  it("opens the meter capture only after recognition owns the microphone", async () => {
+    const getUserMedia = stubSpeechEnvironment(
+      vi.fn().mockResolvedValue({ getTracks: () => [] }),
+    );
+    const probe = await renderVoiceDictation();
+    await act(async () => {
+      await probe.latest().start();
+    });
+
+    expect(getUserMedia).not.toHaveBeenCalled();
+    await act(async () => {
+      FakeSpeechRecognition.last!.onaudiostart?.();
+    });
+    expect(getUserMedia).toHaveBeenCalledTimes(1);
+    await probe.cleanup();
+  });
+});

package/src/composer/useVoiceDictation.ts

@@ -348,7 +348,18 @@ export function voiceDictationStartErrorMessage(error: unknown): string {
   return message || "Could not start recording";
 }
 
-function voiceDictationSpeechErrorMessage(error: string | undefined): string {
+/** Retrying these through another provider re-prompts and fails the same way. */
+function isMicPermissionError(error: string | undefined): boolean {
+  return (
+    error === "not-allowed" ||
+    error === "service-not-allowed" ||
+    error === "audio-capture"
+  );
+}
+
+export function voiceDictationSpeechErrorMessage(
+  error: string | undefined,
+): string {
   if (error === "not-allowed" || error === "service-not-allowed") {
     return voiceDictationStartErrorMessage({
       name: "NotAllowedError",
@@ -358,7 +369,13 @@ function voiceDictationSpeechErrorMessage(error: string | undefined): string {
   if (error === "audio-capture") {
     return "No microphone was found. Plug one in or choose a different input, then try again.";
   }
-  return `Speech recognition error: ${error ?? "unknown"}`;
+  if (error === "network") {
+    return "Speech recognition couldn't reach its service. Check your connection, or pick a different source in Settings → Voice Transcription.";
+  }
+  if (error === "aborted" || error === undefined) {
+    return "Dictation stopped before it captured any audio. Another app or tab may be holding the microphone — close it, or pick a different source in Settings → Voice Transcription.";
+  }
+  return `Speech recognition error: ${error}`;
 }
 
 export function useVoiceDictation(
@@ -688,28 +705,20 @@ export function useVoiceDictation(
   );
 
   const startBrowser = useCallback(
-    async (prefs: VoicePrefs) => {
+    async (
+      prefs: VoicePrefs,
+      /** Return true to take over when the recognizer never opened the mic.
+       * Brave ships `webkitSpeechRecognition` with no speech backend, so
+       * feature detection alone cannot tell dictation will work. */
+      onUnavailable?: (error: string | undefined) => boolean,
+    ) => {
       const Ctor = getSpeechRecognitionCtor();
       if (!Ctor) {
         throw new Error(
           "Your browser doesn't support speech recognition. Add an OpenAI API key in settings for Whisper transcription.",
         );
       }
-      // Still request mic to drive the amplitude meter, so the UI doesn't look
-      // dead while the user talks. SpeechRecognition manages its own capture
-      // under the hood in most browsers.
-      let stream: MediaStream | null = null;
-      try {
-        stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-        mediaStreamRef.current = stream;
-        startMeter(stream);
-      } catch {
-        /* non-fatal — recognition can still work without our analyser */
-      }
-
       if (cancelledRef.current) {
-        if (stream) for (const track of stream.getTracks()) track.stop();
-        mediaStreamRef.current = null;
         cancelledRef.current = false;
         setState("idle");
         return;
@@ -723,7 +732,34 @@ export function useVoiceDictation(
       speechRef.current = recognition;
       speechTranscriptRef.current = "";
 
+      let capturing = false;
+      let fatal = false;
+      let lastError: string | undefined;
+
+      // Opening our own capture before the speech service has claimed the
+      // device makes Chrome abort the session outright. Attach the meter only
+      // once recognition is actually listening.
+      recognition.onaudiostart = () => {
+        capturing = true;
+        if (cancelledRef.current) return;
+        void Promise.resolve()
+          .then(() => navigator.mediaDevices?.getUserMedia({ audio: true }))
+          .then((stream) => {
+            if (!stream) return;
+            if (cancelledRef.current || speechRef.current !== recognition) {
+              for (const track of stream.getTracks()) track.stop();
+              return;
+            }
+            mediaStreamRef.current = stream;
+            startMeter(stream);
+          })
+          .catch(() => {
+            /* the meter is decoration; recognition owns the real capture */
+          });
+      };
+
       recognition.onresult = (event: any) => {
+        capturing = true;
         let interim = "";
         for (let i = event.resultIndex; i < event.results.length; i++) {
           const result = event.results[i];
@@ -736,16 +772,30 @@ export function useVoiceDictation(
         }
         onLiveUpdateRef.current?.(speechTranscriptRef.current, interim);
       };
+      // `end` always follows `error`, so every outcome is decided there. Acting
+      // here too would either pre-empt the fallback or be overwritten by it.
       recognition.onerror = (event: any) => {
+        lastError = event?.error;
         if (event?.error === "no-speech" || event?.error === "aborted") return;
-        failWith(voiceDictationSpeechErrorMessage(event?.error));
+        fatal = true;
       };
       recognition.onend = () => {
         const text = speechTranscriptRef.current.trim();
         const wasCancelled = cancelledRef.current;
         cancelledRef.current = false;
         teardown();
-        if (wasCancelled || !text) {
+        if (wasCancelled) {
+          setState("idle");
+          return;
+        }
+        if (!text) {
+          // A recognizer that failed, or that ended before the mic ever opened,
+          // produced nothing usable — that is not the same as hearing silence.
+          if (fatal || !capturing) {
+            if (onUnavailable?.(lastError)) return;
+            failWith(voiceDictationSpeechErrorMessage(lastError));
+            return;
+          }
           setState("idle");
           return;
         }
@@ -1082,7 +1132,22 @@ export function useVoiceDictation(
         }
         await startGoogleRealtime(prefs);
       } else {
-        await startBrowser(prefs);
+        // Only "auto" promised a working recognizer of any kind; an explicit
+        // browser preference must surface its own failure instead.
+        await startBrowser(
+          prefs,
+          pref === "auto" && mediaRecorderSupported
+            ? (error) => {
+                if (isMicPermissionError(error)) return false;
+                activeProviderRef.current = "openai";
+                setState("starting");
+                void startOpenAi("auto", prefs.instructions).catch((err) =>
+                  failWith(voiceDictationStartErrorMessage(err)),
+                );
+                return true;
+              }
+            : undefined,
+        );
       }
     } catch (err) {
       if (cancelledRef.current) {