@agent-native/core 0.114.12 → 0.114.15

package/corpus/templates/clips/desktop/src/hooks/useWhisperSettings.ts

@@ -0,0 +1,172 @@
+import { invoke } from "@tauri-apps/api/core";
+import { listen } from "@tauri-apps/api/event";
+import { useCallback, useEffect, useState } from "react";
+
+import type { VoiceProvider } from "../lib/voice-dictation";
+import type { FeatureConfig } from "../shared/config";
+
+export interface WhisperModelOption {
+  id: string;
+  title: string;
+  description: string;
+  url: string;
+  filename: string;
+  sha256: string;
+  sizeBytes: number;
+  sizeMb: number;
+}
+
+export type WhisperModelState =
+  | "disabled"
+  | "missing"
+  | "downloading"
+  | "ready";
+
+export interface WhisperModelStatus {
+  state: WhisperModelState;
+  path: string;
+  downloadedMb: number;
+  totalMb: number;
+}
+
+const WHISPER_STATUS_EVENTS = [
+  "whisper:model-progress",
+  "whisper:model-error",
+  "whisper:model-enabled-changed",
+  "whisper:model-selection-changed",
+] as const;
+
+export function useWhisperSettings(
+  featureConfig: FeatureConfig | null,
+  voiceProvider: VoiceProvider,
+  onVoiceProviderChange: (provider: VoiceProvider) => void,
+  nativeVoiceProvider: () => VoiceProvider,
+) {
+  const [catalog, setCatalog] = useState<WhisperModelOption[]>([]);
+  const [downloadedModelIds, setDownloadedModelIds] = useState<string[]>([]);
+  const [status, setStatus] = useState<WhisperModelStatus | null>(null);
+
+  const enabled = featureConfig?.whisperModelEnabled !== false;
+  const modelId = featureConfig?.whisperModelId ?? "base";
+  const selectedModel = catalog.find((m) => m.id === modelId) ?? null;
+  const deletableModels = catalog.filter(
+    (m) => m.id !== modelId && downloadedModelIds.includes(m.id),
+  );
+
+  useEffect(() => {
+    invoke<WhisperModelOption[]>("whisper_models")
+      .then(setCatalog)
+      .catch(() => {});
+  }, []);
+
+  const refreshDownloaded = useCallback(() => {
+    invoke<string[]>("whisper_downloaded_models")
+      .then(setDownloadedModelIds)
+      .catch(() => {});
+  }, []);
+
+  useEffect(() => {
+    let cancelled = false;
+    const refreshStatus = () => {
+      invoke<WhisperModelStatus>("whisper_model_status")
+        .then((s) => {
+          if (!cancelled) setStatus(s);
+        })
+        .catch(() => {});
+    };
+
+    refreshStatus();
+    refreshDownloaded();
+
+    const unlistens: Array<() => void> = [];
+    const track = (p: Promise<() => void>) => {
+      p.then((u) => {
+        if (cancelled) {
+          try {
+            u();
+          } catch {
+            /* ignore */
+          }
+          return;
+        }
+        unlistens.push(u);
+      }).catch(() => {});
+    };
+
+    for (const event of WHISPER_STATUS_EVENTS) {
+      track(listen(event, () => refreshStatus()));
+    }
+    track(
+      listen("whisper:model-ready", () => {
+        refreshStatus();
+        refreshDownloaded();
+      }),
+    );
+    track(listen("whisper:model-deleted", () => refreshDownloaded()));
+
+    return () => {
+      cancelled = true;
+      unlistens.forEach((u) => {
+        try {
+          u();
+        } catch {
+          /* ignore */
+        }
+      });
+    };
+  }, [refreshDownloaded]);
+
+  function triggerDownload() {
+    invoke("whisper_model_download").catch(() => {});
+  }
+
+  function setEnabled(next: boolean) {
+    if (!featureConfig) return;
+    invoke("set_feature_config", {
+      config: { ...featureConfig, whisperModelEnabled: next },
+    })
+      .then(() => {
+        if (next) {
+          triggerDownload();
+        } else if (voiceProvider === "whisper") {
+          onVoiceProviderChange(nativeVoiceProvider());
+        }
+      })
+      .catch((err) =>
+        console.error("[whisper] set_feature_config failed", err),
+      );
+  }
+
+  function setModelId(next: string) {
+    if (!featureConfig) return;
+    invoke("set_feature_config", {
+      config: { ...featureConfig, whisperModelId: next },
+    })
+      .then(() => {
+        if (enabled) triggerDownload();
+      })
+      .catch((err) =>
+        console.error("[whisper] set_feature_config failed", err),
+      );
+  }
+
+  function deleteModel(modelIdToDelete: string) {
+    invoke("whisper_model_delete", { modelId: modelIdToDelete })
+      .then(() => refreshDownloaded())
+      .catch(() => {});
+  }
+
+  return {
+    catalog,
+    downloadedModelIds,
+    status,
+    enabled,
+    modelId,
+    selectedModel,
+    deletableModels,
+    triggerDownload,
+    setEnabled,
+    setModelId,
+    deleteModel,
+  };
+}

package/corpus/templates/clips/desktop/src/lib/meeting-call-app.ts

@@ -0,0 +1,41 @@
+const NATIVE_CALL_APP_BUNDLE_IDS = [
+  "us.zoom.xos",
+  "us.zoom.ZoomClips",
+  "com.microsoft.teams2",
+  "com.microsoft.teams",
+] as const;
+
+const BROWSER_CALL_APP_BUNDLE_IDS = [
+  "com.google.Chrome",
+  "company.thebrowser.Browser",
+  "com.apple.Safari",
+  "org.mozilla.firefox",
+] as const;
+
+function isHost(hostname: string, host: string): boolean {
+  return hostname === host || hostname.endsWith(`.${host}`);
+}
+
+/**
+ * Limit microphone-use monitoring to the app that can own this meeting.
+ * Browser bundles are included only when the calendar join URL identifies a
+ * browser-hosted provider, so another tab using the microphone cannot end a
+ * native Zoom or Teams recording.
+ */
+export function callAppBundleIdsForJoinUrl(joinUrl?: string | null): string[] {
+  if (!joinUrl) return [...NATIVE_CALL_APP_BUNDLE_IDS];
+
+  try {
+    const hostname = new URL(joinUrl).hostname.toLowerCase();
+    if (isHost(hostname, "meet.google.com")) {
+      return [...BROWSER_CALL_APP_BUNDLE_IDS];
+    }
+    if (isHost(hostname, "teams.microsoft.com")) {
+      return [...NATIVE_CALL_APP_BUNDLE_IDS, ...BROWSER_CALL_APP_BUNDLE_IDS];
+    }
+  } catch {
+    // Native Zoom and Teams remain the safe fallback for malformed URLs.
+  }
+
+  return [...NATIVE_CALL_APP_BUNDLE_IDS];
+}

package/corpus/templates/clips/desktop/src/lib/silence-events.ts

@@ -23,6 +23,7 @@ export interface SilenceConfig {
   silenceThreshold?: number;
   silenceMs?: number;
   callEndedMs?: number;
+  callAppBundleIds?: string[];
   scheduledEndMs?: number | null;
   watchSleep?: boolean;
   watchCallEnded?: boolean;

package/corpus/templates/clips/desktop/src/lib/voice-dictation.ts

@@ -1445,6 +1445,9 @@ export function installDesktopVoiceDictation(
         // the tail because Web Speech only marks a segment as `isFinal`
         // after a confidence-threshold pass.
         next.browserTranscript = (finalSoFar + interim).trim();
+        emit("voice:dictation-preview", {
+          text: next.browserTranscript,
+        }).catch(() => {});
       };
       recognition.onerror = (ev) => {
         if (ev.error !== "no-speech" && ev.error !== "aborted") {
@@ -2038,15 +2041,16 @@ export function installDesktopVoiceDictation(
 
   // Native (SFSpeechRecognizer) event subscriptions. These are always
   // installed — the events only fire when the Rust side has an active
-  // session, so subscribing on non-native sessions is harmless. The
-  // flow-bar listens to `voice:partial-transcript` independently so we
-  // don't re-emit it here.
+  // session, so subscribing on non-native sessions is harmless.
   onPartialTranscript(({ text }) => {
     const current = session;
     if (!current || (current.kind !== "native" && current.kind !== "whisper"))
       return;
     if (current.cancelled || current.stopping) return;
     setInterimTranscript(current, text);
+    emit("voice:dictation-preview", {
+      text: current.browserTranscript,
+    }).catch(() => {});
   })
     .then((u) => unlistens.push(u))
     .catch(() => {});
@@ -2065,6 +2069,13 @@ export function installDesktopVoiceDictation(
     if (!current) return;
     if (current.cancelled) return;
     appendFinalTranscript(current, text);
+    const supersededLingeringSession =
+      current === lingeringSession && session !== null && session !== current;
+    if (!supersededLingeringSession) {
+      emit("voice:dictation-preview", {
+        text: current.browserTranscript,
+      }).catch(() => {});
+    }
     if (current === lingeringSession) {
       current.onNativeFinalize?.();
     }

package/corpus/templates/clips/desktop/src/overlays/flow-bar.tsx

@@ -16,6 +16,7 @@ type FlowState = "idle" | "recording" | "processing" | "complete" | "error";
  * Events:
  *   - `voice:state-change` { state: "idle"|"recording"|"processing"|"complete"|"error" }
  *   - `voice:audio-level` { level: number } (0-1) for waveform visualization
+ *   - `voice:dictation-preview` { text: string }
  */
 export function FlowBar() {
   // Default to "recording" not "idle" — there's a race between the Rust
@@ -23,6 +24,8 @@ export function FlowBar() {
   // "idle" caused the bar to flash an "EN" language pill that never went
   // away if the start event was missed.
   const [state, setState] = useState<FlowState>("recording");
+  const [transcript, setTranscript] = useState("");
+  const transcriptRef = useRef<HTMLDivElement | null>(null);
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const levelRef = useRef(0);
   const rafRef = useRef<number | null>(null);
@@ -48,6 +51,7 @@ export function FlowBar() {
     trackListen(
       listen<{ state: FlowState }>("voice:state-change", (ev) => {
         setState(ev.payload.state);
+        if (ev.payload.state === "recording") setTranscript("");
       }),
     );
 
@@ -57,6 +61,12 @@ export function FlowBar() {
       }),
     );
 
+    trackListen(
+      listen<{ text: string }>("voice:dictation-preview", (ev) => {
+        setTranscript(ev.payload.text.trim());
+      }),
+    );
+
     return () => {
       stopped = true;
       unlistens.forEach((u) => {
@@ -70,6 +80,11 @@ export function FlowBar() {
     };
   }, []);
 
+  useEffect(() => {
+    const preview = transcriptRef.current;
+    if (preview) preview.scrollTop = preview.scrollHeight;
+  }, [transcript]);
+
   // Waveform canvas rendering loop — only runs during the "recording" state.
   useEffect(() => {
     if (state !== "recording") {
@@ -151,6 +166,16 @@ export function FlowBar() {
 
   return (
     <div className="flow-bar-root">
+      {transcript ? (
+        <div
+          ref={transcriptRef}
+          className="flow-bar-transcript-preview"
+          aria-live="polite"
+        >
+          {transcript}
+        </div>
+      ) : null}
+
       {/* Pill is ALWAYS mounted — when state goes idle we fade the
           opacity to 0 (see CSS) instead of removing it from the DOM.
           Inner content keeps its last frame rendered during the fade

package/corpus/templates/clips/desktop/src/shared/config.ts

@@ -130,6 +130,7 @@ export interface FeatureConfig {
   screenMemory: ScreenMemoryConfig;
   onboardingComplete: boolean;
   whisperModelEnabled: boolean;
+  whisperModelId: string;
 }
 
 export function useFeatureConfig() {

package/corpus/templates/clips/desktop/src/styles.css

@@ -4905,13 +4905,33 @@ body[data-clips-route="recording-pill"] #root {
   position: fixed;
   inset: var(--overlay-shadow-gutter);
   display: flex;
+  flex-direction: column;
   align-items: center;
   justify-content: flex-end;
+  gap: 6px;
   padding-bottom: 8px;
   background: transparent;
   pointer-events: none;
 }
 
+.flow-bar-transcript-preview {
+  max-width: 560px;
+  max-height: 90px;
+  padding: 5px 10px;
+  overflow: hidden;
+  border: 1px solid rgba(255, 255, 255, 0.06);
+  border-radius: 10px;
+  background: rgba(18, 18, 20, 0.92);
+  color: rgba(255, 255, 255, 0.9);
+  font-size: 12px;
+  font-weight: 500;
+  line-height: 18px;
+  text-align: center;
+  word-break: break-word;
+  backdrop-filter: blur(20px);
+  -webkit-backdrop-filter: blur(20px);
+}
+
 .flow-bar {
   display: flex;
   align-items: center;
@@ -5580,23 +5600,39 @@ body[data-clips-route="recording-pill"] #root {
 }
 
 .whisper-progress-bar {
-  height: 4px;
   width: 100%;
+  height: 4px;
+  margin-top: 4px;
+  border: none;
   border-radius: 999px;
-  background: var(--surface-strong);
+  appearance: none;
   overflow: hidden;
-  margin-top: 4px;
+  background: var(--surface-strong);
+  color: #16a34a;
 }
 
-.whisper-progress-fill {
-  height: 100%;
+.whisper-progress-bar::-webkit-progress-bar {
+  border-radius: inherit;
+  background: var(--surface-strong);
+}
+
+.whisper-progress-bar::-webkit-progress-value {
+  border-radius: inherit;
+  background: #16a34a;
+}
+
+.whisper-progress-bar::-moz-progress-bar {
   border-radius: inherit;
   background: #16a34a;
-  transition: width 300ms ease;
 }
 
 @media (prefers-color-scheme: dark) {
-  .whisper-progress-fill {
+  .whisper-progress-bar {
+    color: #4ade80;
+  }
+
+  .whisper-progress-bar::-webkit-progress-value,
+  .whisper-progress-bar::-moz-progress-bar {
     background: #4ade80;
   }
 }
@@ -5624,6 +5660,63 @@ body[data-clips-route="recording-pill"] #root {
   background: var(--surface-hover);
 }
 
+.whisper-other-models {
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+  margin-top: 6px;
+}
+
+.whisper-other-model-row {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 8px;
+  padding: 5px 8px;
+  border: 1px solid var(--border);
+  border-radius: var(--radius-sm);
+  background: var(--surface);
+}
+
+.whisper-other-model-name {
+  font-size: 11px;
+  color: var(--fg-muted);
+}
+
+.whisper-delete-btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  flex-shrink: 0;
+  height: 22px;
+  padding: 0 8px;
+  border: 1px solid var(--border-strong);
+  border-radius: var(--radius-sm);
+  background: var(--surface);
+  color: var(--fg-muted);
+  cursor: pointer;
+  font-size: 11px;
+  font-weight: 600;
+  transition:
+    background 120ms,
+    color 120ms,
+    border-color 120ms;
+}
+
+.whisper-delete-btn:hover {
+  background: rgba(239, 68, 68, 0.1);
+  border-color: rgba(239, 68, 68, 0.4);
+  color: #ef4444;
+}
+
+@media (prefers-color-scheme: dark) {
+  .whisper-delete-btn:hover {
+    background: rgba(239, 68, 68, 0.15);
+    border-color: rgba(239, 68, 68, 0.5);
+    color: #f87171;
+  }
+}
+
 /* ---- */
 
 .pill-error {

package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs

@@ -1506,10 +1506,9 @@ pub async fn show_flow_bar(app: AppHandle) -> Result<(), String> {
 
     let (mx, my, mw, mh) = tray_monitor_physical_rect(&app);
     let scale = overlay_scale_factor(&app);
-    // Compact Wispr-style pill window: just enough transparent canvas for
-    // the bottom-centered waveform bar and its shadow gutter.
-    let content_w: u32 = (160.0 * scale).round() as u32;
-    let content_h: u32 = (56.0 * scale).round() as u32;
+    // Wide + tall enough for a 5-line transcript preview above the pill.
+    let content_w: u32 = (640.0 * scale).round() as u32;
+    let content_h: u32 = (160.0 * scale).round() as u32;
     let bottom_margin: i32 = (14.0 * scale).round() as i32;
     let gutter = overlay_shadow_gutter_physical(&app);
     let w: u32 = content_w + gutter * 2;

package/corpus/templates/clips/desktop/src-tauri/src/config.rs

@@ -135,6 +135,8 @@ pub struct FeatureConfig {
     pub onboarding_complete: bool,
     #[serde(default = "default_whisper_model_enabled")]
     pub whisper_model_enabled: bool,
+    #[serde(default = "default_whisper_model_id")]
+    pub whisper_model_id: String,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -170,6 +172,10 @@ fn default_whisper_model_enabled() -> bool {
     true
 }
 
+fn default_whisper_model_id() -> String {
+    "base".to_string()
+}
+
 fn default_screen_memory_retention_hours() -> u32 {
     8
 }
@@ -227,6 +233,7 @@ impl Default for FeatureConfig {
             screen_memory: ScreenMemoryConfig::default(),
             onboarding_complete: false,
             whisper_model_enabled: default_whisper_model_enabled(),
+            whisper_model_id: default_whisper_model_id(),
         }
     }
 }
@@ -333,6 +340,12 @@ pub async fn get_feature_config(app: AppHandle) -> Result<FeatureConfig, String>
 /// Save feature config to disk and emit a change event.
 #[tauri::command]
 pub async fn set_feature_config(app: AppHandle, config: FeatureConfig) -> Result<(), String> {
+    if !crate::whisper_model::is_supported_model_id(&config.whisper_model_id) {
+        return Err(format!(
+            "unsupported Whisper model: {}",
+            config.whisper_model_id
+        ));
+    }
     let previous = load_config(&app);
     if (crate::rewind_clip::is_active(&app) || crate::util::is_recording_active(&app))
         && rewind_capture_contract_changed(&previous.screen_memory, &config.screen_memory)
@@ -366,6 +379,12 @@ pub async fn set_feature_config(app: AppHandle, config: FeatureConfig) -> Result
             serde_json::json!({ "enabled": config.whisper_model_enabled }),
         );
     }
+    if previous.whisper_model_id != config.whisper_model_id {
+        let _ = app.emit(
+            "whisper:model-selection-changed",
+            serde_json::json!({ "modelId": config.whisper_model_id }),
+        );
+    }
     crate::screen_memory::sync_from_config(&app, &config);
     let _ = app.emit("app:feature-config-changed", config);
     Ok(())
@@ -406,6 +425,19 @@ mod tests {
             .contains(&"com.1password.1password".to_string()));
     }
 
+    #[test]
+    fn feature_config_defaults_to_base_whisper_model_for_legacy_json() {
+        let config: FeatureConfig = serde_json::from_value(serde_json::json!({
+            "clipsEnabled": true,
+            "meetingsEnabled": true,
+            "voiceEnabled": true,
+            "onboardingComplete": true
+        }))
+        .unwrap();
+
+        assert_eq!(config.whisper_model_id, "base");
+    }
+
     #[test]
     fn rewind_enums_use_stable_kebab_case_values() {
         assert_eq!(

package/corpus/templates/clips/desktop/src-tauri/src/lib.rs

@@ -231,8 +231,11 @@ pub fn run() {
             shortcuts::set_fn_shortcut_enabled,
             shortcuts::set_dictation_escape_active,
             // whisper model management
+            whisper_model::whisper_models,
             whisper_model::whisper_model_status,
             whisper_model::whisper_model_download,
+            whisper_model::whisper_downloaded_models,
+            whisper_model::whisper_model_delete,
             // permission status (silent checks for all TCC permissions)
             permission_status::check_permission_statuses,
             // persistent log file (production debugging)

package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs

@@ -2465,8 +2465,15 @@ impl LiveAudioMixer {
     }
 
     /// Emit mixed sample buffers covering `out_pos..safe_end` in
-    /// `MIX_CHUNK_FRAMES` chunks: sum system + mic per frame, clamp, wrap as
-    /// LPCM `CMSampleBuffer`s with contiguous PTS.
+    /// `MIX_CHUNK_FRAMES` chunks: average system + mic per frame, clamp, wrap
+    /// as LPCM `CMSampleBuffer`s with contiguous PTS.
+    ///
+    /// Each source is weighted at 0.5 before summing so that two full-scale
+    /// signals (which occurs when a USB audio interface with software monitoring
+    /// routes the mic back through system audio) can never exceed ±1.0 and
+    /// hard-clip. The standard SCK pipeline applies the same 0.5×L + 0.5×R
+    /// pan-downmix for the same reason; loudnorm restores the target loudness
+    /// in post-processing.
     fn drain_ready(
         &mut self,
         flush: bool,
@@ -2488,8 +2495,8 @@ impl LiveAudioMixer {
                 let frame = a + f as i64;
                 let (sl, sr) = self.system.sample_at(frame);
                 let (ml, mr) = self.mic.sample_at(frame);
-                interleaved[f * 2] = (sl + ml).clamp(-1.0, 1.0);
-                interleaved[f * 2 + 1] = (sr + mr).clamp(-1.0, 1.0);
+                interleaved[f * 2] = (sl * 0.5 + ml * 0.5).clamp(-1.0, 1.0);
+                interleaved[f * 2 + 1] = (sr * 0.5 + mr * 0.5).clamp(-1.0, 1.0);
             }
             emitted.push(self.build_sample_buffer(&interleaved, a)?);
             a = b;