@agent-native/core 0.92.9 → 0.92.11

package/corpus/templates/clips/desktop/src-tauri/src/system_audio.rs

@@ -25,8 +25,8 @@
 //! | `audio_transcription_reset_timeline`| Rebase transcript timestamps to now.     |
 //! | `audio_transcription_stop`         | Stop the capture.                         |
 //!
-//! `start_raw_system_capture` (in the `macos` submodule) is the capture entry
-//! point the Whisper engine calls directly.
+//! `start_raw_system_capture` and `start_raw_meeting_capture` (in the `macos`
+//! submodule) are the capture entry points the Whisper engine calls directly.
 //!
 //! ## Events
 //!   - `voice:audio-level` `{ level, source: "system" }` — waveform meter.
@@ -124,7 +124,10 @@ pub async fn audio_transcription_start(
         mic_device_id,
         mic_device_label,
         capture_system.unwrap_or(true),
-        voice_processing.unwrap_or(true),
+        // Fail safe for meeting/recording callers: an omitted flag must not
+        // create a second VoiceProcessingIO stack beside a live call app.
+        // Short dictation sessions opt in explicitly from the renderer.
+        voice_processing.unwrap_or(false),
         owner,
     )
     .await
@@ -246,6 +249,8 @@ pub(crate) mod macos {
         app: AppHandle,
         cancelled: Arc<AtomicBool>,
         level_tick: Arc<AtomicU32>,
+        output_type: SCStreamOutputType,
+        source: &'static str,
     }
 
     // SAFETY: `Retained<SFSpeech*>` and `Retained<AVAudioFormat>` wrap
@@ -261,7 +266,7 @@ pub(crate) mod macos {
             sample_buffer: CMSampleBuffer,
             of_type: SCStreamOutputType,
         ) {
-            if of_type != SCStreamOutputType::Audio {
+            if of_type != self.output_type {
                 return;
             }
             if self.cancelled.load(Ordering::SeqCst) {
@@ -287,7 +292,7 @@ pub(crate) mod macos {
                         "voice:audio-level",
                         AudioLevelPayload {
                             level,
-                            source: "system",
+                            source: self.source,
                         },
                     );
                 }
@@ -295,17 +300,18 @@ pub(crate) mod macos {
         }
     }
 
-    /// Handle for a running raw system capture. `stop()` ends the SCK stream.
-    pub(crate) struct RawSystemCapture {
+    /// Handle for a running raw SCK audio capture. A meeting capture can carry
+    /// both system and microphone output handlers on this one stream.
+    pub(crate) struct RawSckAudioCapture {
         stream: SCStream,
         cancelled: Arc<AtomicBool>,
     }
 
     // SAFETY: `SCStream` is `Send` (the crate marks it so); the atomic is
     // trivially `Send`. We only move the handle through ownership.
-    unsafe impl Send for RawSystemCapture {}
+    unsafe impl Send for RawSckAudioCapture {}
 
-    impl RawSystemCapture {
+    impl RawSckAudioCapture {
         pub(crate) fn stop(self) {
             self.cancelled.store(true, Ordering::SeqCst);
             let _ = self.stream.stop_capture();
@@ -317,7 +323,50 @@ pub(crate) mod macos {
     pub(crate) fn start_raw_system_capture(
         app: AppHandle,
         on_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
-    ) -> Result<RawSystemCapture, String> {
+    ) -> Result<RawSckAudioCapture, String> {
+        start_raw_sck_audio_capture(app, true, None, None, Some(on_samples), None)
+    }
+
+    /// Whether this macOS version supports ScreenCaptureKit's independent
+    /// microphone output (`SCStreamOutputType::Microphone`).
+    pub(crate) fn supports_sck_microphone_capture() -> bool {
+        let version = NSProcessInfo::processInfo().operatingSystemVersion();
+        version.majorVersion >= 15
+    }
+
+    /// Start one ScreenCaptureKit stream with independent system-audio and
+    /// microphone callbacks. This avoids opening a second AVAudioEngine /
+    /// VoiceProcessingIO input while Zoom, Meet, or Teams owns the live-call
+    /// microphone uplink.
+    pub(crate) fn start_raw_meeting_capture(
+        app: AppHandle,
+        mic_device_id: Option<String>,
+        mic_device_label: Option<String>,
+        capture_system: bool,
+        on_mic_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
+        on_system_samples: Option<Arc<dyn Fn(&[f32]) + Send + Sync>>,
+    ) -> Result<RawSckAudioCapture, String> {
+        if !supports_sck_microphone_capture() {
+            return Err("ScreenCaptureKit microphone capture requires macOS 15 or later.".into());
+        }
+        start_raw_sck_audio_capture(
+            app,
+            capture_system,
+            mic_device_id,
+            mic_device_label,
+            on_system_samples,
+            Some(on_mic_samples),
+        )
+    }
+
+    fn start_raw_sck_audio_capture(
+        app: AppHandle,
+        capture_system: bool,
+        mic_device_id: Option<String>,
+        mic_device_label: Option<String>,
+        on_system_samples: Option<Arc<dyn Fn(&[f32]) + Send + Sync>>,
+        on_mic_samples: Option<Arc<dyn Fn(&[f32]) + Send + Sync>>,
+    ) -> Result<RawSckAudioCapture, String> {
         let granted = unsafe { CGPreflightScreenCaptureAccess() };
         if !granted {
             let granted_now = unsafe { CGRequestScreenCaptureAccess() };
@@ -340,13 +389,30 @@ pub(crate) mod macos {
             .with_excluding_windows(&[])
             .build();
 
-        let config = SCStreamConfiguration::new()
-            .with_captures_audio(true)
+        let capture_microphone = on_mic_samples.is_some();
+        let selected_mic = if capture_microphone {
+            crate::native_screen::resolve_microphone_capture_device(
+                mic_device_id.as_deref(),
+                mic_device_label.as_deref(),
+            )?
+        } else {
+            None
+        };
+        let mut config = SCStreamConfiguration::new()
+            .with_captures_audio(capture_system)
+            .with_captures_microphone(capture_microphone)
             .with_excludes_current_process_audio(true)
             .with_sample_rate(48000)
             .with_channel_count(2)
             .with_width(2)
             .with_height(2);
+        if let Some(device) = selected_mic.as_ref() {
+            config.set_microphone_capture_device_id(&device.id);
+            eprintln!(
+                "[whisper] ScreenCaptureKit meeting microphone pinned to {} ({})",
+                device.name, device.id
+            );
+        }
 
         // Mono float32 @ 48 kHz destination format for the mono-mix.
         let speech_format = unsafe {
@@ -357,21 +423,37 @@ pub(crate) mod macos {
 
         let cancelled = Arc::new(AtomicBool::new(false));
         let mut stream = SCStream::new(&filter, &config);
-        let forwarder = RawAudioForwarder {
-            on_samples,
-            speech_format,
-            app: app.clone(),
-            cancelled: cancelled.clone(),
-            level_tick: Arc::new(AtomicU32::new(0)),
-        };
-        stream.add_output_handler(forwarder, SCStreamOutputType::Audio);
+        if let Some(on_samples) = on_system_samples {
+            let forwarder = RawAudioForwarder {
+                on_samples,
+                speech_format: speech_format.clone(),
+                app: app.clone(),
+                cancelled: cancelled.clone(),
+                level_tick: Arc::new(AtomicU32::new(0)),
+                output_type: SCStreamOutputType::Audio,
+                source: "system",
+            };
+            stream.add_output_handler(forwarder, SCStreamOutputType::Audio);
+        }
+        if let Some(on_samples) = on_mic_samples {
+            let forwarder = RawAudioForwarder {
+                on_samples,
+                speech_format,
+                app: app.clone(),
+                cancelled: cancelled.clone(),
+                level_tick: Arc::new(AtomicU32::new(0)),
+                output_type: SCStreamOutputType::Microphone,
+                source: "mic",
+            };
+            stream.add_output_handler(forwarder, SCStreamOutputType::Microphone);
+        }
 
         if let Err(e) = stream.start_capture() {
             cancelled.store(true, Ordering::SeqCst);
             return Err(format!("SCStream start_capture failed: {e:?}"));
         }
 
-        Ok(RawSystemCapture { stream, cancelled })
+        Ok(RawSckAudioCapture { stream, cancelled })
     }
 
     #[derive(Serialize, Clone)]

package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs

@@ -10,7 +10,9 @@
 //! Capture is reused from the existing modules:
 //!   - mic    → `native_speech::macos::start_raw_mic_capture` (AVAudioEngine +
 //!              optional VoiceProcessingIO AEC, other-audio ducking off)
-//!   - system → `system_audio::macos::start_raw_system_capture` (ScreenCaptureKit)
+//!   - meetings on macOS 15+ → one ScreenCaptureKit stream with independent
+//!              microphone + system-audio outputs
+//!   - legacy system audio → `system_audio::macos::start_raw_system_capture`
 //!
 use tauri::AppHandle;
 
@@ -108,7 +110,10 @@ mod macos {
     use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
 
     use crate::native_speech::macos::{start_raw_mic_capture, RawMicCapture};
-    use crate::system_audio::macos::{start_raw_system_capture, RawSystemCapture};
+    use crate::system_audio::macos::{
+        start_raw_meeting_capture, start_raw_system_capture, supports_sck_microphone_capture,
+        RawSckAudioCapture,
+    };
     use crate::whisper_model::{ensure_model, model_file};
 
     /// One transcript segment with real timestamps from whisper, already
@@ -571,11 +576,20 @@ mod macos {
         }
     }
 
+    fn should_use_combined_sck_capture(
+        owner: SessionOwner,
+        microphone_capture_supported: bool,
+    ) -> bool {
+        owner == SessionOwner::Meeting && microphone_capture_supported
+    }
+
     struct Session {
-        mic_cap: RawMicCapture,
+        // macOS 15+ meetings use a combined SCK capture, so there is no
+        // competing AVAudioEngine / VoiceProcessingIO mic input to stop.
+        mic_cap: Option<RawMicCapture>,
         // System capture is optional — skipped when the user turns system
         // audio off, so neither the recording nor the transcript include it.
-        sys_cap: Option<RawSystemCapture>,
+        sys_cap: Option<RawSckAudioCapture>,
         mic: Arc<WhisperStream>,
         sys: Option<Arc<WhisperStream>>,
         /// Who started this session — see `SessionOwner`.
@@ -644,8 +658,10 @@ mod macos {
         let _ = language;
         let lang: Option<String> = None;
 
-        // Mic stream + capture. The real hardware rate is read back from the
-        // capture handle and pushed into the stream (default 48 kHz until then).
+        // Create both Whisper streams first. On macOS 15+ meetings, one
+        // ScreenCaptureKit stream feeds both callbacks without opening a
+        // competing VoiceProcessingIO mic input. Older macOS versions (and a
+        // failed SCK start) keep the existing split-capture fallback.
         let session_start = Instant::now();
         let mic_stream = WhisperStream::new(
             app.clone(),
@@ -655,50 +671,92 @@ mod macos {
             ctx.clone(),
             session_start,
         );
-        let mic_for_cb = mic_stream.clone();
-        let reuse_voice_processing_engine = owner == SessionOwner::Dictation && !capture_system;
-        let mic_cap = start_raw_mic_capture(
-            app.clone(),
-            mic_device_id,
-            mic_device_label,
-            voice_processing,
-            reuse_voice_processing_engine,
-            Arc::new(move |s: &[f32]| mic_for_cb.push(s)),
-        )
-        .map_err(|e| {
-            mic_stream.stop();
-            format!("mic capture failed: {e}")
-        })?;
-        mic_stream.set_src_rate(mic_cap.sample_rate());
-
-        // System stream + capture (SCK delivers 48 kHz). Skipped entirely when
-        // system audio is off.
-        let (sys_stream, sys_cap) = if capture_system {
-            let sys_stream = WhisperStream::new(
+        let sys_stream = capture_system.then(|| {
+            WhisperStream::new(
                 app.clone(),
                 "system",
                 48000.0,
                 lang.clone(),
                 ctx.clone(),
                 session_start,
-            );
-            let sys_for_cb = sys_stream.clone();
-            let sys_cap = match start_raw_system_capture(
+            )
+        });
+        let mic_for_cb = mic_stream.clone();
+        let mic_callback: Arc<dyn Fn(&[f32]) + Send + Sync> =
+            Arc::new(move |samples: &[f32]| mic_for_cb.push(samples));
+        let system_callback: Option<Arc<dyn Fn(&[f32]) + Send + Sync>> =
+            sys_stream.as_ref().map(|stream| {
+                let stream = stream.clone();
+                Arc::new(move |samples: &[f32]| stream.push(samples))
+                    as Arc<dyn Fn(&[f32]) + Send + Sync>
+            });
+
+        let combined_cap = if should_use_combined_sck_capture(
+            owner,
+            supports_sck_microphone_capture(),
+        ) {
+            match start_raw_meeting_capture(
                 app.clone(),
-                Arc::new(move |s: &[f32]| sys_for_cb.push(s)),
+                mic_device_id.clone(),
+                mic_device_label.clone(),
+                capture_system,
+                mic_callback.clone(),
+                system_callback.clone(),
             ) {
-                Ok(cap) => cap,
+                Ok(cap) => {
+                    eprintln!("[whisper] using combined ScreenCaptureKit mic + system capture");
+                    Some(cap)
+                }
                 Err(e) => {
-                    // Roll back the mic side so we don't leave a half-open meeting.
+                    eprintln!(
+                        "[whisper] combined ScreenCaptureKit meeting capture failed: {e}; falling back to split capture"
+                    );
+                    None
+                }
+            }
+        } else {
+            None
+        };
+
+        let (mic_cap, sys_cap) = if let Some(combined_cap) = combined_cap {
+            // Both SCK outputs are configured at 48 kHz.
+            mic_stream.set_src_rate(48000.0);
+            (None, Some(combined_cap))
+        } else {
+            let reuse_voice_processing_engine = owner == SessionOwner::Dictation && !capture_system;
+            let mic_cap = start_raw_mic_capture(
+                app.clone(),
+                mic_device_id,
+                mic_device_label,
+                voice_processing,
+                reuse_voice_processing_engine,
+                mic_callback,
+            )
+            .map_err(|e| {
+                mic_stream.stop();
+                if let Some(sys_stream) = &sys_stream {
                     sys_stream.stop();
-                    mic_stream.stop();
-                    mic_cap.stop();
-                    return Err(format!("system capture failed: {e}"));
                 }
+                format!("mic capture failed: {e}")
+            })?;
+            mic_stream.set_src_rate(mic_cap.sample_rate());
+
+            let sys_cap = if let Some(system_callback) = system_callback {
+                match start_raw_system_capture(app.clone(), system_callback) {
+                    Ok(cap) => Some(cap),
+                    Err(e) => {
+                        if let Some(sys_stream) = &sys_stream {
+                            sys_stream.stop();
+                        }
+                        mic_stream.stop();
+                        mic_cap.stop();
+                        return Err(format!("system capture failed: {e}"));
+                    }
+                }
+            } else {
+                None
             };
-            (Some(sys_stream), Some(sys_cap))
-        } else {
-            (None, None)
+            (Some(mic_cap), sys_cap)
         };
 
         let mut slot = session_slot().lock().map_err(|e| e.to_string())?;
@@ -750,7 +808,9 @@ mod macos {
             sys.stop();
         }
         // Stop captures so no more samples arrive while workers flush.
-        session.mic_cap.stop();
+        if let Some(mic_cap) = session.mic_cap {
+            mic_cap.stop();
+        }
         if let Some(sys_cap) = session.sys_cap {
             sys_cap.stop();
         }
@@ -769,4 +829,22 @@ mod macos {
         }
         eprintln!("[whisper] meeting transcription stopped");
     }
+
+    #[cfg(test)]
+    mod tests {
+        use super::{should_use_combined_sck_capture, SessionOwner};
+
+        #[test]
+        fn combined_sck_capture_is_only_selected_for_supported_meetings() {
+            assert!(should_use_combined_sck_capture(SessionOwner::Meeting, true));
+            assert!(!should_use_combined_sck_capture(
+                SessionOwner::Meeting,
+                false
+            ));
+            assert!(!should_use_combined_sck_capture(
+                SessionOwner::Dictation,
+                true
+            ));
+        }
+    }
 }

package/corpus/templates/clips/server/lib/slack-unfurls.ts

@@ -28,6 +28,7 @@ type SlackUnfurlRecording = {
   id: string;
   title: string;
   description: string;
+  durationMs: number;
   thumbnailUrl: string | null;
   animatedThumbnailUrl: string | null;
   visibility: string | null;
@@ -205,6 +206,19 @@ function isExpired(value: string | null): boolean {
   return Number.isFinite(expires) && expires < Date.now();
 }
 
+function formatSlackDuration(durationMs: number): string | null {
+  if (!Number.isFinite(durationMs) || durationMs <= 0) return null;
+  const totalSeconds = Math.floor(durationMs / 1000);
+  const hours = Math.floor(totalSeconds / 3600);
+  const minutes = Math.floor((totalSeconds % 3600) / 60);
+  const seconds = totalSeconds % 60;
+  return hours > 0
+    ? `${hours}:${minutes.toString().padStart(2, "0")}:${seconds
+        .toString()
+        .padStart(2, "0")}`
+    : `${minutes}:${seconds.toString().padStart(2, "0")}`;
+}
+
 export function isSlackPlayableRecording(
   recording: SlackUnfurlRecording,
 ): boolean {
@@ -236,7 +250,11 @@ export function buildSlackVideoBlock(options: {
     origin,
   ).toString();
   const title = displayRecordingTitle(recording.title);
-  const description = clipsShareDescription(recording);
+  const shareDescription = clipsShareDescription(recording);
+  const duration = formatSlackDuration(recording.durationMs);
+  const description = duration
+    ? `${duration} · ${shareDescription}`
+    : shareDescription;
   const thumbnailUrl = absoluteUrl(
     recording.thumbnailUrl || recording.animatedThumbnailUrl,
     origin,
@@ -265,6 +283,7 @@ export async function loadSlackVideoBlockForUrl(
       id: schema.recordings.id,
       title: schema.recordings.title,
       description: schema.recordings.description,
+      durationMs: schema.recordings.durationMs,
       thumbnailUrl: schema.recordings.thumbnailUrl,
       animatedThumbnailUrl: schema.recordings.animatedThumbnailUrl,
       visibility: schema.recordings.visibility,

package/corpus/templates/clips/server/lib/streaming-upload-mode.ts

@@ -1,8 +1,8 @@
 import { enabledFlag } from "./env-flags.js";
 
-// Streaming resumable uploads are deployment opt-in while the provider/finalize
-// path hardens. Set CLIPS_ENABLE_STREAMING_UPLOAD=1 to allow recorder requests;
-// CLIPS_DISABLE_STREAMING_UPLOAD=1 still forces the buffered fallback.
+// Local/dev deployments can opt into resumable uploads while retaining their
+// SQL scratch fallback. Hosted deployments have no safe buffered fallback, so
+// requested video uploads use resumable storage unless explicitly disabled.
 export function isStreamingUploadDisabled(): boolean {
   return enabledFlag(process.env.CLIPS_DISABLE_STREAMING_UPLOAD);
 }
@@ -10,9 +10,15 @@ export function isStreamingUploadDisabled(): boolean {
 export function shouldEnableStreamingUpload(args: {
   client?: string | null;
   mimeType?: string | null;
+  bufferedFallbackAvailable?: boolean;
 }): boolean {
   if (isStreamingUploadDisabled()) return false;
-  if (!enabledFlag(process.env.CLIPS_ENABLE_STREAMING_UPLOAD)) return false;
+  if (
+    args.bufferedFallbackAvailable !== false &&
+    !enabledFlag(process.env.CLIPS_ENABLE_STREAMING_UPLOAD)
+  ) {
+    return false;
+  }
 
   const mimeType = (args.mimeType ?? "").split(";")[0]?.trim().toLowerCase();
   return !mimeType || mimeType.startsWith("video/");

package/corpus/templates/clips/shared/recording-core.ts

@@ -51,6 +51,18 @@ export function pickMimeType(): string {
  *  - `"buffered"`  — full blob assembled after stop() and uploaded in slices */
 export type UploadMode = "streaming" | "buffered";
 
+/**
+ * Resumable providers advance a single byte offset, so their chunks must be
+ * sent in strict index order. Buffered uploads can retain bounded parallelism.
+ */
+export function chunkUploadParallelism(
+  uploadMode: UploadMode | undefined,
+  bufferedParallelism: number,
+): number {
+  if (uploadMode === "streaming") return 1;
+  return Math.max(1, Math.floor(bufferedParallelism));
+}
+
 /** Query params understood by the chunk-upload route
  * (`/api/uploads/:id/chunk`). This is the on-the-wire contract — the route in
  * `server/routes/api/uploads/[recordingId]/chunk.post.ts` reads exactly these. */

package/corpus/templates/content/AGENTS.md

@@ -158,13 +158,14 @@ to SQL. `pull-document` closes that gap with a flush handshake — if a live Yjs
 collab session exists for the document it writes a one-shot `flush-request-<id>`
 application-state key (scoped to the browser session, just like `navigate`); the
 open editor sees that key, serializes its current document to markdown through
-its own existing serializer, calls `update-document`, and deletes the key;
-`pull-document` waits for the key to clear and then returns the now-fresh row.
-When no editor is open the SQL column is authoritative and the handshake is
+its own existing serializer, calls `update-document`, and writes an explicit
+request-id-matched success/error acknowledgement. `pull-document` waits for that
+acknowledgement and fails closed if the open editor cannot save; when no active
+human editor is present, the SQL column is authoritative and the handshake is
 skipped. It is GET + read-only + public-agent exposed (`requiresAuth: true`),
-returns `{ id, title, content, format, deepLink }`, and surfaces an
-"Open document" deep link for external agents. Use `--format text` for a
-plain-text strip of the markdown.
+returns `{ id, title, content, format, deepLink }`, and surfaces an "Open
+document" deep link for external agents. Use `--format text` for a plain-text
+strip of the markdown.
 
 ### Local Source Files