@agent-native/core 0.87.0 → 0.90.1

package/corpus/templates/clips/desktop/src-tauri/src/native_speech.rs

@@ -35,17 +35,29 @@ pub async fn native_speech_start(
     locale: Option<String>,
     mic_device_id: Option<String>,
     mic_device_label: Option<String>,
+    owner: Option<String>,
 ) -> Result<(), String> {
     #[cfg(target_os = "macos")]
     {
         // contextual_strings (personal vocabulary) is staged separately via
         // `native_speech_set_vocabulary` so mic metadata can flow through
         // meeting capture without coupling vocabulary into that path.
-        macos::native_speech_start_impl(app, locale, mic_device_id, mic_device_label)
+        //
+        // `owner` defaults to "dictation" for back-compat with callers that
+        // don't pass it. Meetings pass "meeting" (transcription-engine.ts) so
+        // a meeting's native-speech session can refuse a dictation takeover —
+        // see `SessionOwner` / the cancel-prior-session check below.
+        macos::native_speech_start_impl(
+            app,
+            locale,
+            mic_device_id,
+            mic_device_label,
+            macos::SessionOwner::from_param(owner),
+        )
     }
     #[cfg(not(target_os = "macos"))]
     {
-        let _ = (app, locale, mic_device_id, mic_device_label);
+        let _ = (app, locale, mic_device_id, mic_device_label, owner);
         Err("Native speech recognition is only supported on macOS.".into())
     }
 }
@@ -141,6 +153,29 @@ pub(crate) mod macos {
 
     use screencapturekit::audio_devices::AudioInputDevice;
 
+    /// Who owns an in-flight `SpeechSession`. Meetings fall back to this
+    /// mic-only engine when whisper fails; a Fn/dictation press must not be
+    /// able to silently kill a meeting's live capture (D10). Priority rule:
+    /// meeting beats dictation. All other combinations (same owner
+    /// replacing itself, or a meeting evicting a dictation session) keep the
+    /// original unconditional cancel+replace behavior.
+    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
+    pub(crate) enum SessionOwner {
+        Dictation,
+        Meeting,
+    }
+
+    impl SessionOwner {
+        /// Parses the Tauri command's `owner` string param, defaulting to
+        /// `Dictation` for back-compat with callers that omit it.
+        pub(crate) fn from_param(owner: Option<String>) -> Self {
+            match owner.as_deref() {
+                Some("meeting") => SessionOwner::Meeting,
+                _ => SessionOwner::Dictation,
+            }
+        }
+    }
+
     /// One in-flight dictation. Holds strong references to the AppKit objects
     /// so they don't drop while the recognition task is still emitting
     /// results.
@@ -168,6 +203,8 @@ pub(crate) mod macos {
         /// installed; we swap this to false on the first removal and skip
         /// subsequent calls.
         tap_installed: AtomicBool,
+        /// Who started this session — see `SessionOwner`.
+        owner: SessionOwner,
     }
 
     // SAFETY: see the doc comment on `SpeechSession`. We never alias the
@@ -190,6 +227,13 @@ pub(crate) mod macos {
         GEN.get_or_init(|| AtomicU64::new(0))
     }
 
+    /// Cap on consecutive transient-error auto-restarts (see
+    /// `native_speech_start_impl`'s `restart_attempt` param) before we give up
+    /// and surface `voice:speech-error` instead of retrying forever. A
+    /// persistently silent mic (codes 203/1110) would otherwise restart every
+    /// ~300ms indefinitely.
+    const MAX_TRANSIENT_RESTARTS: u32 = 5;
+
     #[derive(Serialize, Clone)]
     struct PartialPayload {
         text: String,
@@ -217,40 +261,90 @@ pub(crate) mod macos {
         pub source: &'static str,
     }
 
-    /// Cheap peak-magnitude meter over channel 0 of a PCM buffer. Returns a
+    /// Cheap peak-magnitude meter across all channels of a PCM buffer. Returns a
     /// value in `0..=1`. Used by both the mic tap (here) and the system-audio
     /// tap (in `system_audio.rs`) to drive the dual-stream waveform.
     pub(crate) fn peak_level_for_pcm(buf: &AVAudioPCMBuffer) -> f32 {
         // SAFETY: AVAudioPCMBuffer with float format exposes `floatChannelData`
         // as a pointer to `channelCount` pointers, each pointing at
-        // `frameLength` floats. We only read channel 0, bounded by the
+        // `frameLength` floats. We read each channel, bounded by the
         // engine-reported frame length.
         unsafe {
             let frames = buf.frameLength() as usize;
             if frames == 0 {
                 return 0.0;
             }
+            let channel_count = buf.format().channelCount() as usize;
+            if channel_count == 0 {
+                return 0.0;
+            }
             let channels_ptr = buf.floatChannelData();
             if channels_ptr.is_null() {
                 return 0.0;
             }
-            let ch0 = (*channels_ptr).as_ptr();
-            let slice = std::slice::from_raw_parts(ch0, frames);
             let mut peak: f32 = 0.0;
             // Sample sparsely — we don't need every frame for a meter.
             let step = (frames / 64).max(1);
-            let mut i = 0;
-            while i < frames {
-                let v = slice[i].abs();
-                if v > peak {
-                    peak = v;
+            for channel in 0..channel_count {
+                let channel_ptr = (*channels_ptr.add(channel)).as_ptr();
+                if channel_ptr.is_null() {
+                    continue;
+                }
+                let slice = std::slice::from_raw_parts(channel_ptr, frames);
+                let mut i = 0;
+                while i < frames {
+                    let v = slice[i].abs();
+                    if v > peak {
+                        peak = v;
+                    }
+                    i += step;
                 }
-                i += step;
             }
             peak.min(1.0)
         }
     }
 
+    fn mono_mix_pcm(buf: &AVAudioPCMBuffer) -> Vec<f32> {
+        // SAFETY: Same buffer layout as `peak_level_for_pcm`. Multi-channel
+        // built-in Mac microphones can put useful voice energy outside channel
+        // 0, so mix all channels for Whisper instead of forwarding only the
+        // first channel.
+        unsafe {
+            let frames = buf.frameLength() as usize;
+            if frames == 0 {
+                return Vec::new();
+            }
+            let channel_count = buf.format().channelCount() as usize;
+            if channel_count == 0 {
+                return Vec::new();
+            }
+            let channels_ptr = buf.floatChannelData();
+            if channels_ptr.is_null() {
+                return Vec::new();
+            }
+            let mut mono = vec![0.0_f32; frames];
+            let mut mixed_channels = 0usize;
+            for channel in 0..channel_count {
+                let channel_ptr = (*channels_ptr.add(channel)).as_ptr();
+                if channel_ptr.is_null() {
+                    continue;
+                }
+                let slice = std::slice::from_raw_parts(channel_ptr, frames);
+                for (dst, src) in mono.iter_mut().zip(slice.iter()) {
+                    *dst += *src;
+                }
+                mixed_channels += 1;
+            }
+            if mixed_channels > 1 {
+                let scale = 1.0 / mixed_channels as f32;
+                for sample in &mut mono {
+                    *sample *= scale;
+                }
+            }
+            mono
+        }
+    }
+
     /// Block synchronously until the system has a definitive authorization
     /// decision. Returns the final status. The handler block runs on an
     /// internal queue, so we use a one-shot mpsc channel to bridge it back
@@ -567,7 +661,40 @@ pub(crate) mod macos {
         locale: Option<String>,
         mic_device_id: Option<String>,
         mic_device_label: Option<String>,
+        owner: SessionOwner,
+    ) -> Result<(), String> {
+        native_speech_start_impl_inner(app, locale, mic_device_id, mic_device_label, owner, 0)
+    }
+
+    /// `restart_attempt` counts consecutive transient-error auto-restarts of
+    /// the *same* logical session lineage (see `MAX_TRANSIENT_RESTARTS`). Every
+    /// externally-visible entry point (the public `native_speech_start_impl`
+    /// above) passes `0`; only the auto-restart thread below increments it.
+    /// The auto-restart thread always passes the original session's `owner`
+    /// through unchanged, so a meeting's own transient-error restart is never
+    /// misclassified as a foreign dictation takeover.
+    fn native_speech_start_impl_inner(
+        app: AppHandle,
+        locale: Option<String>,
+        mic_device_id: Option<String>,
+        mic_device_label: Option<String>,
+        owner: SessionOwner,
+        restart_attempt: u32,
     ) -> Result<(), String> {
+        // Priority rule (D10): a meeting-owned session must never be
+        // silently evicted by a dictation takeover. Check (without taking)
+        // BEFORE bumping the generation or touching the slot, so a refused
+        // dictation start leaves the meeting's session — and its own
+        // transient-restart lineage — completely untouched.
+        {
+            let slot = session_slot().lock().map_err(|e| e.to_string())?;
+            if let Some(prev) = slot.as_ref() {
+                if prev.owner == SessionOwner::Meeting && owner == SessionOwner::Dictation {
+                    return Err("speech-engine-busy-meeting".into());
+                }
+            }
+        }
+
         // Bump the generation so any pending auto-restart for the previous
         // session's transient error will see the counter has changed and abort.
         let my_gen = session_generation().fetch_add(1, Ordering::SeqCst) + 1;
@@ -577,7 +704,9 @@ pub(crate) mod macos {
             (!v.is_empty()).then_some(v)
         };
         // Cancel any prior session first — there's only one mic tap per input
-        // node, and we want a deterministic state going in.
+        // node, and we want a deterministic state going in. (Any other
+        // owner combination — same-owner replacement, or meeting evicting
+        // dictation — keeps this unconditional cancel+replace behavior.)
         {
             let mut slot = session_slot().lock().map_err(|e| e.to_string())?;
             if let Some(prev) = slot.take() {
@@ -745,19 +874,30 @@ pub(crate) mod macos {
                         err.code()
                     );
 
-                    if !is_cancelled && !transient {
+                    let restarts_exhausted =
+                        transient && restart_attempt + 1 >= MAX_TRANSIENT_RESTARTS;
+
+                    if !is_cancelled && (!transient || restarts_exhausted) {
+                        let error = if restarts_exhausted {
+                            format!(
+                                "Speech recognition kept failing ({msg}) — stopped after {MAX_TRANSIENT_RESTARTS} attempts."
+                            )
+                        } else {
+                            msg
+                        };
                         let _ = app.emit(
                             "voice:speech-error",
                             ErrorPayload {
-                                error: msg,
+                                error,
                                 source: "mic",
                             },
                         );
                     }
                     clear_session_slot();
 
-                    if !is_cancelled && !is_stopped && transient {
+                    if !is_cancelled && !is_stopped && transient && !restarts_exhausted {
                         let gen = my_gen;
+                        let next_attempt = restart_attempt + 1;
                         let app = app.clone();
                         let locale = locale.clone();
                         let mic_device_id = mic_device_id.clone();
@@ -768,11 +908,13 @@ pub(crate) mod macos {
                             if session_generation().load(Ordering::SeqCst) != gen {
                                 return;
                             }
-                            if let Err(e) = native_speech_start_impl(
+                            if let Err(e) = native_speech_start_impl_inner(
                                 app.clone(),
                                 locale,
                                 mic_device_id,
                                 mic_device_label,
+                                owner,
+                                next_attempt,
                             ) {
                                 let _ = app.emit(
                                     "voice:speech-error",
@@ -834,6 +976,7 @@ pub(crate) mod macos {
                 cancelled,
                 stopped,
                 tap_installed: AtomicBool::new(true),
+                owner,
             });
         }
 
@@ -869,12 +1012,14 @@ pub(crate) mod macos {
         }
     }
 
-    /// Start mic capture (VPIO AEC on, other-audio ducking off) and forward
-    /// every mono channel-0 f32 buffer to `on_samples`.
+    /// Start mic capture and forward a mono mix of every available channel to
+    /// `on_samples`. Recording transcription can disable VPIO so the tap does
+    /// not precondition the shared mic before ScreenCaptureKit opens it.
     pub(crate) fn start_raw_mic_capture(
         app: AppHandle,
         mic_device_id: Option<String>,
         mic_device_label: Option<String>,
+        voice_processing: bool,
         on_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
     ) -> Result<RawMicCapture, String> {
         let engine: Retained<AVAudioEngine> = unsafe { AVAudioEngine::new() };
@@ -885,15 +1030,17 @@ pub(crate) mod macos {
         )?;
         let input_node: Retained<objc2_avf_audio::AVAudioInputNode> = unsafe { engine.inputNode() };
 
-        unsafe {
-            if let Err(err) = input_node.setVoiceProcessingEnabled_error(true) {
+        if voice_processing {
+            if let Err(err) = unsafe { input_node.setVoiceProcessingEnabled_error(true) } {
                 eprintln!(
                     "[whisper-mic] voice processing enable failed: {} — continuing without AEC",
                     ns_error_message(&err)
                 );
             }
+        } else {
+            eprintln!("[whisper-mic] voice processing disabled for shared mic capture");
         }
-        // Finalize VPIO format negotiation.
+        // Finalize input format negotiation.
         objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe { engine.prepare() }))
             .map_err(|e| format!("AVAudioEngine prepare threw: {e:?}"))?;
 
@@ -905,14 +1052,9 @@ pub(crate) mod macos {
                 move |buffer: std::ptr::NonNull<AVAudioPCMBuffer>,
                       _when: std::ptr::NonNull<AVAudioTime>| {
                     let buf = unsafe { buffer.as_ref() };
-                    let frames = unsafe { buf.frameLength() } as usize;
-                    if frames > 0 {
-                        let ch_ptr = unsafe { buf.floatChannelData() };
-                        if !ch_ptr.is_null() {
-                            let slice =
-                                unsafe { std::slice::from_raw_parts((*ch_ptr).as_ptr(), frames) };
-                            on_samples(slice);
-                        }
+                    let mono = mono_mix_pcm(buf);
+                    if !mono.is_empty() {
+                        on_samples(&mono);
                     }
                     let n = level_tick.fetch_add(1, Ordering::Relaxed);
                     if n % 2 == 0 {
@@ -959,20 +1101,22 @@ pub(crate) mod macos {
             unsafe { format.channelCount() }
         );
 
-        // Disable other-audio ducking so the separately-captured system audio
-        // isn't dropped to near-silent while the mic VPIO runs.
-        unsafe {
-            let responds: bool = objc2::msg_send![
-                &*input_node,
-                respondsToSelector: objc2::sel!(setVoiceProcessingOtherAudioDuckingConfiguration:)
-            ];
-            if responds {
-                input_node.setVoiceProcessingOtherAudioDuckingConfiguration(
-                    AVAudioVoiceProcessingOtherAudioDuckingConfiguration {
-                        enableAdvancedDucking: objc2::runtime::Bool::NO,
-                        duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingLevel::Min,
-                    },
-                );
+        if voice_processing {
+            // Disable other-audio ducking so the separately-captured system audio
+            // isn't dropped to near-silent while the mic VPIO runs.
+            unsafe {
+                let responds: bool = objc2::msg_send![
+                    &*input_node,
+                    respondsToSelector: objc2::sel!(setVoiceProcessingOtherAudioDuckingConfiguration:)
+                ];
+                if responds {
+                    input_node.setVoiceProcessingOtherAudioDuckingConfiguration(
+                        AVAudioVoiceProcessingOtherAudioDuckingConfiguration {
+                            enableAdvancedDucking: objc2::runtime::Bool::NO,
+                            duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingLevel::Min,
+                        },
+                    );
+                }
             }
         }