@agent-native/core 0.76.12 → 0.76.13

package/corpus/templates/clips/desktop/src/app.tsx

@@ -295,7 +295,7 @@ function installAuthFetchInterceptor(): void {
 }
 
 type ByokVoiceProvider = Extract<VoiceProvider, "gemini" | "groq">;
-type VoiceProviderMode = "native" | "builder" | "byok";
+type VoiceProviderMode = "native" | "whisper" | "builder" | "byok";
 type MacosPrivacyPane =
   | "camera"
   | "microphone"
@@ -368,6 +368,7 @@ function isByokVoiceProvider(value: VoiceProvider): value is ByokVoiceProvider {
 function voiceProviderMode(value: VoiceProvider): VoiceProviderMode {
   if (isByokVoiceProvider(value)) return "byok";
   if (value === "builder" || value === "builder-gemini") return "builder";
+  if (value === "whisper") return "whisper";
   return "native";
 }
 
@@ -378,6 +379,7 @@ function normalizeVoiceProvider(value: string): VoiceProvider {
   if (value === "macos-native" && !isMacPlatform()) return "browser";
   return value === "browser" ||
     value === "macos-native" ||
+    value === "whisper" ||
     value === "builder-gemini" ||
     value === "gemini" ||
     value === "groq"
@@ -2987,7 +2989,11 @@ function Setup({
     }).catch((err) =>
       console.error("[settings] set_feature_config failed", err),
     );
-    if (enabled) triggerWhisperDownload();
+    if (enabled) {
+      triggerWhisperDownload();
+    } else if (voiceProvider === "whisper") {
+      onVoiceProviderChange(nativeVoiceProvider());
+    }
   }
 
   function setLaunchAtLoginEnabled(enabled: boolean) {
@@ -3223,6 +3229,7 @@ function Setup({
     native: isMacPlatform()
       ? "Uses macOS on-device speech recognition for the fastest free dictation."
       : "Uses the browser's built-in speech recognition when available.",
+    whisper: "Uses the local Whisper model for offline AI transcription.",
     builder:
       "Uses Builder.io for fast cleanup. No separate provider key needed.",
     byok: "Use your own provider key for cleanup.",
@@ -3245,6 +3252,9 @@ function Setup({
     setApiKeyMessage(null);
     if (mode === "native") {
       onVoiceProviderChange(nativeVoiceProvider());
+    } else if (mode === "whisper") {
+      onVoiceProviderChange("whisper");
+      if (!whisperModelEnabled) setWhisperModelEnabled(true);
     } else if (mode === "builder") {
       onVoiceProviderChange("builder-gemini");
     } else {
@@ -3333,6 +3343,7 @@ function Setup({
   const providerWarning: string | null = (() => {
     if (providerStatusLoading || !providerStatus) return null;
     if (selectedMode === "native") return null;
+    if (selectedMode === "whisper") return null;
     if (selectedMode === "builder") {
       return providerStatus.builder
         ? null
@@ -3358,6 +3369,8 @@ function Setup({
         <h2>Settings</h2>
       </div>
 
+      <div className="setup-section-heading">General</div>
+
       <div className="setup-section">
         <SettingLabel
           label="Clips server URL"
@@ -3419,6 +3432,23 @@ function Setup({
         </div>
       </div>
 
+      <div className="setup-section-heading">Permissions</div>
+
+      <div className="setup-section">
+        <ReadinessPanel
+          mode="screen-camera"
+          cameraOn={true}
+          micOn={true}
+          includeVoicePaste={voiceEnabled}
+          includeFnMonitoring={fnShortcutSelected}
+          open={readinessOpen}
+          onOpenChange={setReadinessOpen}
+          onOpenPermission={openPrivacySettings}
+        />
+      </div>
+
+      <div className="setup-section-heading">Recording</div>
+
       <details className="setup-advanced">
         <summary className="setup-advanced-summary">Advanced recording</summary>
         <div className="setup-advanced-body">
@@ -3502,19 +3532,26 @@ function Setup({
       </details>
 
       <div className="setup-section">
-        <div className="setup-toggle-row">
-          <SettingLabel
-            label="Voice dictation"
-            hint="Speak to type anywhere on your Mac. Turn off to disable globally and remove the keyboard shortcuts."
-          />
-          <Switch
-            on={voiceEnabled}
-            onChange={setVoiceEnabled}
-            label="Enable voice dictation"
-          />
-        </div>
+        <SettingLabel
+          label="Open Clips shortcut"
+          hint="Optional extra global shortcut for opening the tray popover. Cmd+Shift+L remains available."
+        />
+        <ShortcutRecorder
+          value={popoverCustomShortcut}
+          placeholder="Record shortcut"
+          onChange={onPopoverCustomShortcutChange}
+        />
+        <p className="setup-hint">
+          Use a modifier combination like Cmd+Shift+K. Leave empty to use only
+          Cmd+Shift+L.
+        </p>
+        {shortcutRegistrationError ? (
+          <p className="setup-warning">{shortcutRegistrationError}</p>
+        ) : null}
       </div>
 
+      <div className="setup-section-heading">Meetings</div>
+
       <div className="setup-section">
         <div className="setup-toggle-row">
           <SettingLabel
@@ -3557,25 +3594,6 @@ function Setup({
             </p>
           </div>
 
-          <div className="setup-section">
-            <div className="setup-toggle-row">
-              <SettingLabel
-                label="Whisper model"
-                hint="Local AI model for offline meeting transcription. Captures both your mic and other speakers — no API key required."
-              />
-              <Switch
-                on={whisperModelEnabled}
-                onChange={setWhisperModelEnabled}
-                label="Enable Whisper model"
-              />
-            </div>
-            <WhisperModelStatusRow
-              status={whisperStatus}
-              enabled={whisperModelEnabled}
-              onDownload={triggerWhisperDownload}
-            />
-          </div>
-
           <div className="setup-section">
             <div className="setup-toggle-row">
               <SettingLabel
@@ -3592,40 +3610,41 @@ function Setup({
         </>
       ) : null}
 
+      <div className="setup-section-heading">Whisper</div>
+
       <div className="setup-section">
-        <SettingLabel
-          label="Open Clips shortcut"
-          hint="Optional extra global shortcut for opening the tray popover. Cmd+Shift+L remains available."
-        />
-        <ShortcutRecorder
-          value={popoverCustomShortcut}
-          placeholder="Record shortcut"
-          onChange={onPopoverCustomShortcutChange}
+        <div className="setup-toggle-row">
+          <SettingLabel
+            label="Whisper model"
+            hint="Local AI model for offline transcription (dictation and meetings). No API key required."
+          />
+          <Switch
+            on={whisperModelEnabled}
+            onChange={setWhisperModelEnabled}
+            label="Enable Whisper model"
+          />
+        </div>
+        <WhisperModelStatusRow
+          status={whisperStatus}
+          enabled={whisperModelEnabled}
+          onDownload={triggerWhisperDownload}
         />
-        <p className="setup-hint">
-          Use a modifier combination like Cmd+Shift+K. Leave empty to use only
-          Cmd+Shift+L.
-        </p>
-        {shortcutRegistrationError ? (
-          <p className="setup-warning">{shortcutRegistrationError}</p>
-        ) : null}
       </div>
 
+      <div className="setup-section-heading">Dictation</div>
+
       <div className="setup-section">
-        <SettingLabel
-          label="Privacy permissions"
-          hint="Open the exact Privacy & Security pane for each permission Clips can need."
-        />
-        <ReadinessPanel
-          mode="screen-camera"
-          cameraOn={true}
-          micOn={true}
-          includeVoicePaste={voiceEnabled}
-          includeFnMonitoring={fnShortcutSelected}
-          open={readinessOpen}
-          onOpenChange={setReadinessOpen}
-          onOpenPermission={openPrivacySettings}
-        />
+        <div className="setup-toggle-row">
+          <SettingLabel
+            label="Voice dictation"
+            hint="Speak to type anywhere on your Mac. Turn off to disable globally and remove the keyboard shortcuts."
+          />
+          <Switch
+            on={voiceEnabled}
+            onChange={setVoiceEnabled}
+            label="Enable voice dictation"
+          />
+        </div>
       </div>
 
       {voiceEnabled ? (
@@ -3645,10 +3664,21 @@ function Setup({
               }
             >
               <option value="native">On-device (free, fast)</option>
+              <option value="whisper" disabled={!whisperModelEnabled}>
+                {whisperModelEnabled
+                  ? "Local Whisper (offline AI)"
+                  : "Local Whisper — enable Whisper model first"}
+              </option>
               <option value="builder">Builder.io</option>
               <option value="byok">Add your own key</option>
             </select>
             <p className="setup-hint">{providerHint[selectedMode]}</p>
+            {selectedMode === "whisper" && !whisperModelEnabled ? (
+              <p className="setup-warning">
+                Whisper model is disabled. Enable it in the Whisper section
+                above.
+              </p>
+            ) : null}
             {providerWarning ? (
               <p className="setup-warning">{providerWarning}</p>
             ) : null}
@@ -3697,9 +3727,10 @@ function Setup({
                   }}
                   placeholder={
                     providerStatus?.[byokProvider]
-                      ? "Paste a new key to rotate"
-                      : `Paste ${keyForByokProvider(byokProvider)}`
+                      ? "Key is saved — paste to rotate"
+                      : `Paste ${keyForByokProvider(byokProvider)} here`
                   }
+                  className="setup-key-input"
                 />
                 <button
                   type="button"
@@ -3733,7 +3764,7 @@ function Setup({
             </div>
           ) : null}
 
-          {selectedMode !== "native" ? (
+          {selectedMode !== "native" && selectedMode !== "whisper" ? (
             <div className="setup-section">
               <SettingLabel
                 label="Custom instructions"
@@ -3819,7 +3850,10 @@ function Setup({
           </div>
         </>
       ) : null}
-      <div className="setup-account">
+
+      <div className="setup-section-heading">Debug</div>
+
+      <div className="setup-account setup-account--no-border">
         <button
           type="button"
           className="link-button"
@@ -3906,10 +3940,8 @@ function WhisperModelStatusRow({
     return (
       <div className="whisper-status whisper-status-ready">
         <IconCircleCheck size={13} className="whisper-status-icon" />
-        <span>
-          Ready · {status.totalMb} MB
-          <span className="whisper-status-path">{status.path}</span>
-        </span>
+        <span>Ready · {status.totalMb} MB</span>
+        <span className="whisper-status-path">{status.path}</span>
       </div>
     );
   }

package/corpus/templates/clips/desktop/src/lib/voice-dictation.ts

@@ -32,6 +32,7 @@ export type VoiceProvider =
   | "auto"
   | "browser"
   | "macos-native"
+  | "whisper"
   | "builder-gemini"
   | "builder"
   | "gemini"
@@ -82,8 +83,10 @@ interface VoiceSession {
   // webkitSpeechRecognition (works in Safari and Chromium WebViews,
   // broken in Tauri WKWebView). "native" sessions drive Apple's
   // SFSpeechRecognizer + AVAudioEngine through Tauri commands —
-  // on-device, real-time partials, free, macOS-only.
-  kind: "server" | "browser" | "native";
+  // on-device, real-time partials, free, macOS-only. "whisper" sessions
+  // drive the local whisper.cpp engine via audio_transcription_* commands,
+  // mic-only (captureSystem: false), no API key required.
+  kind: "server" | "browser" | "native" | "whisper";
   // server-only fields
   stream: MediaStream | null;
   recorder: MediaRecorder | null;
@@ -496,6 +499,7 @@ export function installDesktopVoiceDictation(
   const resolveProvider = async (): Promise<
     | { kind: "browser"; cleanupProvider?: ServerVoiceProvider }
     | { kind: "native"; cleanupProvider?: ServerVoiceProvider }
+    | { kind: "whisper"; cleanupProvider?: ServerVoiceProvider }
     | {
         kind: "server";
         providerPref: ServerVoiceProvider;
@@ -515,6 +519,7 @@ export function installDesktopVoiceDictation(
       return { kind: "browser" };
     }
     if (provider === "macos-native") return { kind: "native" };
+    if (provider === "whisper") return { kind: "whisper" };
     if (provider !== "auto") {
       const cleanupProvider =
         provider === "builder" ? "builder-gemini" : provider;
@@ -605,6 +610,8 @@ export function installDesktopVoiceDictation(
         await startBrowser(resolved.cleanupProvider);
       } else if (resolved.kind === "native") {
         await startNative(resolved.cleanupProvider);
+      } else if (resolved.kind === "whisper") {
+        await startWhisper(resolved.cleanupProvider);
       } else {
         await startServer(resolved.providerPref);
       }
@@ -943,6 +950,80 @@ export function installDesktopVoiceDictation(
     }
   };
 
+  /**
+   * Whisper path: local whisper.cpp engine via audio_transcription_* Tauri
+   * commands. Mic-only (captureSystem: false) — same linger/finalize flow
+   * as the native path, same voice:*-transcript events from Rust.
+   */
+  const startWhisper = async (cleanupProvider?: ServerVoiceProvider) => {
+    console.log(
+      "[voice-dictation] startWhisper: invoke audio_transcription_start",
+    );
+    try {
+      // Open the mic BEFORE showing the bar so audio is capturing by the time
+      // the user sees the recording state and starts speaking. The inverse order
+      // (bar first, then start) causes the mic to open ~100-300ms late and the
+      // first spoken words are lost inside audio_transcription_start's
+      // ensure_model + create_state + start_raw_mic_capture sequence.
+      await invoke("audio_transcription_start", {
+        meetingId: null,
+        locale: navigator.language || "en-US",
+        micDeviceId: concreteMediaDeviceId(micDeviceId) || null,
+        micDeviceLabel: micDeviceLabel || null,
+        captureSystem: false,
+      });
+      console.log("[voice-dictation] audio_transcription_start ok");
+      if (disposed || stopRequestedBeforeReady) {
+        invoke("audio_transcription_stop").catch(() => {});
+        abortPendingStart();
+        return;
+      }
+      await invoke("show_flow_bar");
+      if (disposed || stopRequestedBeforeReady) {
+        invoke("audio_transcription_stop").catch(() => {});
+        abortPendingStart();
+        return;
+      }
+      setFlowState("recording");
+      emit("voice:partial-transcript", { text: "" }).catch(() => {});
+      const next: VoiceSession = {
+        kind: "whisper",
+        stream: null,
+        recorder: null,
+        chunks: [],
+        audioContext: null,
+        analyser: null,
+        raf: null,
+        mimeType: "",
+        recognition: null,
+        browserTranscript: "",
+        lastResultAt: 0,
+        startedAt: Date.now(),
+        stopping: false,
+        transcribeAbort: null,
+        cancelled: false,
+        cleanupProvider: cleanupProvider ?? null,
+      };
+      session = next;
+      startInFlight = false;
+      startSyntheticMeter(next);
+      if (stopRequestedBeforeReady) {
+        stop();
+      }
+    } catch (err) {
+      console.error("[voice-dictation] startWhisper failed", err);
+      startInFlight = false;
+      stopRequestedBeforeReady = false;
+      session = null;
+      setFlowState("error");
+      window.setTimeout(() => {
+        if (disposed || session) return;
+        setFlowState("idle");
+        invoke("hide_flow_bar").catch(() => {});
+      }, 800);
+    }
+  };
+
   /**
    * Browser-path: real-time on-device transcription via WKWebView's
    * webkitSpeechRecognition. No server round-trip — text is ready the
@@ -1172,6 +1253,13 @@ export function installDesktopVoiceDictation(
         invoke("native_speech_cancel").catch((err) => {
           console.warn("[voice-dictation] native_speech_cancel failed:", err);
         });
+      } else if (current.kind === "whisper") {
+        invoke("audio_transcription_stop").catch((err) => {
+          console.warn(
+            "[voice-dictation] audio_transcription_stop (cancel) failed:",
+            err,
+          );
+        });
       } else {
         try {
           current.recognition?.abort();
@@ -1225,6 +1313,8 @@ export function installDesktopVoiceDictation(
         }
       } else if (current.kind === "native") {
         invoke("native_speech_cancel").catch(() => {});
+      } else if (current.kind === "whisper") {
+        invoke("audio_transcription_stop").catch(() => {});
       }
       cleanup(current);
       return;
@@ -1232,15 +1322,19 @@ export function installDesktopVoiceDictation(
     try {
       if (current.kind === "server") {
         current.recorder?.stop();
-      } else if (current.kind === "native") {
-        // NATIVE PATH: dismiss the pill *immediately* (snappy UX) but
-        // leave the transcript chip lingering. Tell Rust to `endAudio()`
-        // so SFSpeechRecognizer can deliver its final hypothesis. When
+      } else if (current.kind === "native" || current.kind === "whisper") {
+        // NATIVE / WHISPER PATH: dismiss the pill *immediately* (snappy UX)
+        // but leave the transcript chip lingering. Tell Rust to end the
+        // engine so it can deliver its final hypothesis. When
         // `voice:final-transcript` lands (or after a safety timeout),
         // paste the text and let the chip sit for ~1s with the final
         // word visible — like a notification fading — then dismiss.
-        invoke("native_speech_stop").catch((err) => {
-          console.warn("[voice-dictation] native_speech_stop failed:", err);
+        const stopCmd =
+          current.kind === "whisper"
+            ? "audio_transcription_stop"
+            : "native_speech_stop";
+        invoke(stopCmd).catch((err) => {
+          console.warn(`[voice-dictation] ${stopCmd} failed:`, err);
         });
         // Pill goes RIGHT NOW. The flow-bar window stays open (we'll
         // hide it after the linger) but renders only the transcript
@@ -1576,7 +1670,8 @@ export function installDesktopVoiceDictation(
   // don't re-emit it here.
   onPartialTranscript(({ text }) => {
     const current = session;
-    if (!current || current.kind !== "native") return;
+    if (!current || (current.kind !== "native" && current.kind !== "whisper"))
+      return;
     if (current.cancelled || current.stopping) return;
     current.browserTranscript = text.trim();
   })
@@ -1590,7 +1685,9 @@ export function installDesktopVoiceDictation(
     // late-arriving final from the previous session would otherwise
     // overwrite the new session's transcript with stale text.
     const current =
-      lingeringSession && lingeringSession.kind === "native"
+      lingeringSession &&
+      (lingeringSession.kind === "native" ||
+        lingeringSession.kind === "whisper")
         ? lingeringSession
         : null;
     if (!current) return;
@@ -1607,7 +1704,8 @@ export function installDesktopVoiceDictation(
   onSpeechError(({ error }) => {
     const current = session;
     console.error("[voice-dictation] native speech error:", error);
-    if (!current || current.kind !== "native") return;
+    if (!current || (current.kind !== "native" && current.kind !== "whisper"))
+      return;
     setFlowState("error");
     window.setTimeout(() => {
       if (!disposed && session === current) cleanup(current);

package/corpus/templates/clips/desktop/src/styles.css

@@ -1314,6 +1314,17 @@ body[data-clips-route="recording-pill"] #root {
   gap: 6px;
 }
 
+.setup-section-heading {
+  font-size: 10px;
+  font-weight: 700;
+  letter-spacing: 0.07em;
+  text-transform: uppercase;
+  color: var(--fg-muted);
+  padding-top: 6px;
+  border-top: 1px solid var(--border);
+  margin-top: 4px;
+}
+
 .setup-advanced {
   border: 1px solid var(--border);
   border-radius: var(--radius-sm);
@@ -1638,6 +1649,11 @@ body[data-clips-route="recording-pill"] #root {
   font-size: 12px;
 }
 
+.setup-account--no-border {
+  border-top: none;
+  padding-top: 0;
+}
+
 .setup-account-email {
   color: var(--fg-muted);
   overflow: hidden;
@@ -4200,9 +4216,9 @@ body[data-clips-route="recording-pill"] #root {
 
 .whisper-status {
   display: flex;
-  align-items: flex-start;
+  align-items: center;
   flex-wrap: wrap;
-  gap: 6px;
+  gap: 4px 6px;
   margin-top: 6px;
   font-size: 11px;
   line-height: 1.4;
@@ -4237,8 +4253,7 @@ body[data-clips-route="recording-pill"] #root {
 }
 
 .whisper-status-path {
-  display: block;
-  margin-top: 2px;
+  flex-basis: 100%;
   font-size: 10px;
   color: var(--fg-subtle);
   word-break: break-all;

package/corpus/templates/clips/desktop/src-tauri/Cargo.toml

@@ -69,7 +69,7 @@ libc = "0.2"
 [target."cfg(target_os = \"macos\")".dependencies]
 core-foundation = "0.10"
 core-graphics = "0.25"
-objc2 = "0.6"
+objc2 = { version = "0.6", features = ["exception"] }
 # Native on-device dictation via SFSpeechRecognizer + AVAudioEngine.
 # Powers the "browser" voice-dictation provider (free, instant, no API
 # key, no server round-trip — same engine WisprFlow uses for offline).