@agent-native/core 0.134.0 → 0.135.0

package/corpus/templates/clips/desktop/src-tauri/src/tray.rs

@@ -11,14 +11,59 @@ use crate::tray_meetings::{build_meetings_section, handle_meeting_menu_click, Me
 use crate::util::{is_meeting_active, is_recording_active};
 use crate::TRAY_PNG;
 
-pub fn refresh_tray_anchor(app: &tauri::AppHandle) -> bool {
-    let Some(rect) = app
-        .tray_by_id("main")
-        .and_then(|tray| tray.rect().ok().flatten())
-    else {
+fn physical_tray_rect(rect: tauri::Rect) -> (i32, i32, i32, i32) {
+    let (x, y) = match rect.position {
+        tauri::Position::Physical(position) => (position.x, position.y),
+        tauri::Position::Logical(position) => (position.x as i32, position.y as i32),
+    };
+    let (width, height) = match rect.size {
+        tauri::Size::Physical(size) => (size.width as i32, size.height as i32),
+        tauri::Size::Logical(size) => (size.width as i32, size.height as i32),
+    };
+    (x, y, width, height)
+}
+
+/// A status item that has not been laid out yet reports a frame at the screen
+/// origin, which macOS coordinate flipping turns into a bottom-left rect. Only
+/// a rect sitting in a monitor's menu-bar row can anchor the popover.
+fn tray_rect_is_laid_out(app: &tauri::AppHandle, rect: tauri::Rect) -> bool {
+    let (x, y, width, height) = physical_tray_rect(rect);
+    if width <= 0 || height <= 0 {
+        return false;
+    }
+
+    let center_x = x + width / 2;
+    let center_y = y + height / 2;
+    let Some(window) = app.get_webview_window("popover") else {
+        return false;
+    };
+    let Ok(monitors) = window.available_monitors() else {
         return false;
     };
 
+    monitors.into_iter().any(|monitor| {
+        let position = monitor.position();
+        let size = monitor.size();
+        if center_x < position.x || center_x >= position.x + size.width as i32 {
+            return false;
+        }
+        #[cfg(target_os = "macos")]
+        {
+            let menu_bar_band = (64.0 * monitor.scale_factor()).round() as i32;
+            center_y >= position.y && center_y < position.y + menu_bar_band
+        }
+        #[cfg(not(target_os = "macos"))]
+        {
+            center_y >= position.y && center_y < position.y + size.height as i32
+        }
+    })
+}
+
+fn store_tray_anchor(app: &tauri::AppHandle, rect: tauri::Rect) -> bool {
+    if !tray_rect_is_laid_out(app, rect) {
+        dlog!("[clips-tray] ignoring tray rect that is not laid out yet");
+        return false;
+    }
     let Some(anchor) = app.try_state::<TrayAnchor>() else {
         return false;
     };
@@ -29,6 +74,16 @@ pub fn refresh_tray_anchor(app: &tauri::AppHandle) -> bool {
     true
 }
 
+pub fn refresh_tray_anchor(app: &tauri::AppHandle) -> bool {
+    let Some(rect) = app
+        .tray_by_id("main")
+        .and_then(|tray| tray.rect().ok().flatten())
+    else {
+        return false;
+    };
+    store_tray_anchor(app, rect)
+}
+
 /// Build the full tray menu with the given upcoming-meetings list. Used both
 /// at startup (with `Vec::new()`) and at refresh time when the meetings
 /// watcher pushes a new snapshot — `TrayIcon::set_menu(Some(...))` swaps the
@@ -222,12 +277,7 @@ pub fn build_tray(app: &tauri::App) -> Result<(), Box<dyn std::error::Error>> {
                 _ => None,
             };
             if let Some(rect) = rect {
-                let app = tray.app_handle();
-                if let Some(anchor) = app.try_state::<TrayAnchor>() {
-                    if let Ok(mut g) = anchor.0.lock() {
-                        *g = Some(rect);
-                    }
-                }
+                store_tray_anchor(tray.app_handle(), rect);
             }
 
             if let TrayIconEvent::Click {

package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs

@@ -146,6 +146,7 @@ mod macos {
     use crate::capture_audio_bus::{
         try_subscribe, AudioSources, AudioSubscription, SubscriptionAttempt,
     };
+    use crate::echo_guard::EchoGuard;
     use crate::native_speech::macos::{
         start_raw_mic_capture, MicVoiceProcessingMode, RawMicCapture,
     };
@@ -441,6 +442,11 @@ mod macos {
         /// Recording capture only persists finals and disables this expensive
         /// repeated inference; meeting capture keeps it enabled.
         emit_partials: bool,
+        /// Shared speaker-bleed reference, present only while both streams are
+        /// capturing. The system stream records what the speakers played into
+        /// it; the mic stream reads it back to drop its own echo of that
+        /// playback before inference. See `echo_guard`.
+        echo_guard: Option<Arc<EchoGuard>>,
     }
 
     impl WhisperStream {
@@ -452,6 +458,7 @@ mod macos {
             ctx: Arc<WhisperContext>,
             stream_start: Instant,
             emit_partials: bool,
+            echo_guard: Option<Arc<EchoGuard>>,
         ) -> Arc<Self> {
             let done = Arc::new(AtomicBool::new(false));
             let stream = Arc::new(WhisperStream {
@@ -468,6 +475,7 @@ mod macos {
                 }),
                 reset_generation: AtomicU32::new(0),
                 emit_partials,
+                echo_guard,
             });
             let worker_stream = stream.clone();
             std::thread::spawn(move || {
@@ -492,6 +500,11 @@ mod macos {
             if !self.running.load(Ordering::SeqCst) {
                 return;
             }
+            if self.source == "system" {
+                if let Some(guard) = &self.echo_guard {
+                    guard.note_playback(frames, self.src_rate.load(Ordering::SeqCst) as f64);
+                }
+            }
             if let Ok(mut buf) = self.buf.lock() {
                 buf.extend_from_slice(frames);
             }
@@ -514,15 +527,48 @@ mod macos {
                 .unwrap_or(0)
         }
 
-        /// Mark the start of a fresh buffer (called when the buffer is cleared
+        /// Mark the start of a fresh buffer (called when the buffer is drained
         /// on finalize) so the next utterance's whisper timestamps offset
-        /// correctly onto the meeting timeline.
-        fn reset_buffer_start(&self) {
+        /// correctly onto the meeting timeline. `pending` is how much audio
+        /// survived the drain, which is how far before now the buffer begins.
+        fn reset_buffer_start(&self, pending: Duration) {
+            let now = Instant::now();
             if let Ok(mut timeline) = self.timeline.lock() {
-                timeline.buffer_start = Instant::now();
+                timeline.buffer_start = buffer_start_after_drain(now, pending);
             }
         }
 
+        /// Wall-clock start of the audio currently sitting in `buf`, used to
+        /// line that audio up against the playback reference.
+        fn buffer_start(&self) -> Instant {
+            self.timeline
+                .lock()
+                .map(|timeline| timeline.buffer_start)
+                .unwrap_or_else(|_| Instant::now())
+        }
+
+        /// Whether `samples` (16 kHz mono) is the speakers bleeding into the
+        /// microphone rather than someone talking. Only the mic can be
+        /// contaminated: a call app never plays the local user back.
+        fn is_playback_echo(&self, samples: &[f32]) -> bool {
+            if self.source != "mic" {
+                return false;
+            }
+            self.echo_guard
+                .as_ref()
+                .is_some_and(|guard| guard.is_playback_echo(samples, self.buffer_start()))
+        }
+
+        /// Clear whichever live partial this stream last rendered. Suppressed
+        /// echo emits no final, so without this the overlay would keep showing
+        /// the partial that led up to it.
+        fn clear_partial(&self) {
+            let _ = self.app.emit(
+                "voice:partial-transcript",
+                serde_json::json!({ "text": "", "source": self.source }),
+            );
+        }
+
         /// Rebase timestamps to "now" and discard any audio captured while the
         /// recorder was warming up/counting down.
         fn reset_timeline(&self) {
@@ -682,6 +728,10 @@ mod macos {
         sum / count as f32
     }
 
+    fn buffer_start_after_drain(now: Instant, pending: Duration) -> Instant {
+        now.checked_sub(pending).unwrap_or(now)
+    }
+
     fn partial_inference_due(
         emit_partials: bool,
         had_voice: bool,
@@ -694,6 +744,18 @@ mod macos {
             && since_last_infer > Duration::from_millis(1200)
     }
 
+    fn partial_inference_timestamp(
+        previous: Instant,
+        inference_ran: bool,
+        now: Instant,
+    ) -> Instant {
+        if inference_ran {
+            now
+        } else {
+            previous
+        }
+    }
+
     fn utterance_finalize_due(have_secs: f32, silence: Duration) -> bool {
         (have_secs > 0.4 && silence > Duration::from_millis(800)) || have_secs > 25.0
     }
@@ -790,20 +852,32 @@ mod macos {
                     if stream.reset_generation.load(Ordering::SeqCst) != seen_reset_generation {
                         continue;
                     }
-                    let segs = infer(&mut state, resample_state.samples(), lang);
-                    stream.emit_transcript("voice:final-transcript", &segs, stream.offset_ms());
+                    if stream.is_playback_echo(resample_state.samples()) {
+                        // A dropped utterance is indistinguishable from silence
+                        // in the transcript, so say so here: this log is the
+                        // only way a false positive is diagnosable afterwards.
+                        eprintln!("[whisper-mic] suppressed {have_secs:.1}s of speaker bleed");
+                        stream.clear_partial();
+                    } else {
+                        let segs = infer(&mut state, resample_state.samples(), lang);
+                        stream.emit_transcript("voice:final-transcript", &segs, stream.offset_ms());
+                    }
                 }
+                let mut pending = 0usize;
                 if let Ok(mut b) = stream.buf.lock() {
                     let to_drain = n_processed.min(b.len());
                     b.drain(..to_drain);
+                    pending = b.len();
                 }
                 // Raw indices shift after the drain above (front-truncated),
                 // so the resample cache is invalid regardless of whether this
                 // utterance ran inference — rebuild fresh from whatever's left.
                 resample_state.drop_all();
-                // New buffer begins now — advance the timeline offset so the
-                // next utterance's whisper timestamps map correctly.
-                stream.reset_buffer_start();
+                // Advance the timeline offset so the next utterance's whisper
+                // timestamps map correctly. Inference can take seconds, and
+                // audio kept arriving throughout it, so the new buffer starts
+                // as far back as the audio it already holds — not at "now".
+                stream.reset_buffer_start(Duration::from_secs_f32(pending as f32 / src_rate));
                 last_raw_len = 0;
                 had_voice = false;
                 last_infer = Instant::now();
@@ -825,9 +899,15 @@ mod macos {
                 if stream.reset_generation.load(Ordering::SeqCst) != seen_reset_generation {
                     continue;
                 }
-                let segs = infer(&mut state, resample_state.samples(), lang);
-                stream.emit_transcript("voice:partial-transcript", &segs, stream.offset_ms());
-                last_infer = Instant::now();
+                let inference_ran = if stream.is_playback_echo(resample_state.samples()) {
+                    stream.clear_partial();
+                    false
+                } else {
+                    let segs = infer(&mut state, resample_state.samples(), lang);
+                    stream.emit_transcript("voice:partial-transcript", &segs, stream.offset_ms());
+                    true
+                };
+                last_infer = partial_inference_timestamp(last_infer, inference_ran, Instant::now());
             }
         }
 
@@ -835,7 +915,10 @@ mod macos {
         let raw = stream.buf.lock().map(|b| b.clone()).unwrap_or_default();
         let src_rate = stream.src_rate.load(Ordering::SeqCst) as f64;
         let samples = resample_to_16k(&raw, src_rate);
-        if had_voice && samples.len() as f32 / SAMPLE_RATE_16K > 0.3 {
+        if had_voice
+            && samples.len() as f32 / SAMPLE_RATE_16K > 0.3
+            && !stream.is_playback_echo(&samples)
+        {
             let segs = infer(&mut state, &samples, lang);
             stream.emit_transcript("voice:final-transcript", &segs, stream.offset_ms());
         }
@@ -1028,6 +1111,9 @@ mod macos {
         // competing VoiceProcessingIO mic input. Older macOS versions (and a
         // failed SCK start) keep the existing split-capture fallback.
         let session_start = Instant::now();
+        // Only a session that captures both streams can tell speaker bleed
+        // from speech, so a mic-only session leaves the guard unarmed.
+        let echo_guard = capture_system.then(|| Arc::new(EchoGuard::new()));
         let mic_stream = WhisperStream::new(
             app.clone(),
             "mic",
@@ -1036,6 +1122,7 @@ mod macos {
             ctx.clone(),
             session_start,
             emit_partials,
+            echo_guard.clone(),
         );
         let sys_stream = capture_system.then(|| {
             WhisperStream::new(
@@ -1046,6 +1133,7 @@ mod macos {
                 ctx.clone(),
                 session_start,
                 emit_partials,
+                echo_guard.clone(),
             )
         });
         let mic_for_cb = mic_stream.clone();
@@ -1311,11 +1399,12 @@ mod macos {
 
     #[cfg(test)]
     mod tests {
-        use std::time::Duration;
+        use std::time::{Duration, Instant};
 
         use super::{
-            partial_inference_due, resample_to_16k, should_use_combined_sck_capture,
-            split_mic_capture_options, utterance_finalize_due, IncrementalResample, SessionOwner,
+            buffer_start_after_drain, partial_inference_due, partial_inference_timestamp,
+            resample_to_16k, should_use_combined_sck_capture, split_mic_capture_options,
+            utterance_finalize_due, IncrementalResample, SessionOwner,
         };
         use crate::native_speech::macos::MicVoiceProcessingMode;
 
@@ -1364,6 +1453,17 @@ mod macos {
             );
         }
 
+        #[test]
+        fn buffer_start_accounts_for_audio_captured_during_inference() {
+            let now = Instant::now();
+            let pending = Duration::from_millis(750);
+
+            assert_eq!(
+                buffer_start_after_drain(now, pending),
+                now.checked_sub(pending).unwrap()
+            );
+        }
+
         #[test]
         fn recording_mode_never_runs_live_partial_inference() {
             assert!(!partial_inference_due(
@@ -1390,6 +1490,15 @@ mod macos {
             ));
         }
 
+        #[test]
+        fn echo_suppressed_partial_does_not_reset_retry_cadence() {
+            let previous = Instant::now();
+            let now = previous + Duration::from_secs(2);
+
+            assert_eq!(partial_inference_timestamp(previous, false, now), previous);
+            assert_eq!(partial_inference_timestamp(previous, true, now), now);
+        }
+
         #[test]
         fn recording_mode_keeps_silence_and_long_utterance_finalization() {
             assert!(utterance_finalize_due(1.0, Duration::from_millis(801)));

package/corpus/templates/clips/desktop/vitest.config.ts

@@ -0,0 +1,9 @@
+import baseConfig from "@agent-native/core/vitest-config";
+import { mergeConfig } from "vitest/config";
+
+import viteConfig from "./vite.config";
+
+// Kept separate from vite.config.ts so the production config never imports
+// vitest: `agent-native build` loads vite.config.ts in installs where vitest
+// is absent.
+export default mergeConfig(viteConfig, baseConfig);

package/corpus/templates/clips/server/db/schema.ts

@@ -146,6 +146,16 @@ export const recordings = table("recordings", {
   thumbnailUrl: text("thumbnail_url"),
   animatedThumbnailUrl: text("animated_thumbnail_url"),
 
+  // Editor timeline filmstrip: one sprite image plus the grid geometry needed
+  // to address a cell. Null means "not generated yet", which is why the editor
+  // still has a browser-side extraction fallback.
+  filmstripUrl: text("filmstrip_url"),
+  filmstripFrameCount: integer("filmstrip_frame_count").notNull().default(0),
+  filmstripColumns: integer("filmstrip_columns").notNull().default(0),
+  filmstripRows: integer("filmstrip_rows").notNull().default(0),
+  filmstripFrameWidth: integer("filmstrip_frame_width").notNull().default(0),
+  filmstripFrameHeight: integer("filmstrip_frame_height").notNull().default(0),
+
   durationMs: integer("duration_ms").notNull().default(0),
   videoUrl: text("video_url"),
   videoFormat: text("video_format", { enum: ["webm", "mp4"] })
@@ -168,6 +178,13 @@ export const recordings = table("recordings", {
   // Authoritative liveness for an in-flight upload: renewed by every chunk
   // POST, and the only thing the upload reaper is allowed to consult.
   uploadLeaseExpiresAt: text("upload_lease_expires_at"),
+  // Fences resumed writers: every recovery claim rotates this token so stale
+  // chunks and delayed interruption callbacks cannot mutate the new attempt.
+  uploadAttemptId: text("upload_attempt_id"),
+  // Every destructive restart receives a new generation. Unlike the attempt
+  // id (which is deliberately stable across a lost response), this fences the
+  // provider handle and buffered scratch that the restart replaces.
+  uploadGenerationId: text("upload_generation_id"),
   failureReason: text("failure_reason"),
   loomImportClaimId: text("loom_import_claim_id"),
   loomImportClaimedAt: text("loom_import_claimed_at"),

package/corpus/templates/clips/server/jobs/media-verification.ts

@@ -67,6 +67,7 @@ export async function runMediaVerificationSweepOnce(): Promise<void> {
         .select({
           ownerEmail: schema.recordings.ownerEmail,
           orgId: schema.recordings.orgId,
+          uploadGenerationId: schema.recordings.uploadGenerationId,
         })
         .from(schema.recordings)
         .where(
@@ -92,6 +93,9 @@ export async function runMediaVerificationSweepOnce(): Promise<void> {
               MAX_ATTEMPTS,
               marker.completedAttempts + 1,
             ),
+            ...(recording.uploadGenerationId
+              ? { uploadGenerationId: recording.uploadGenerationId }
+              : {}),
           });
         },
       );

package/corpus/templates/clips/server/lib/activity-notifications.ts

@@ -0,0 +1,185 @@
+/**
+ * Email notifications for Activity events (comments, replies, reactions).
+ *
+ * Recipient resolution and delivery reporting live in
+ * `@agent-native/core/server`; this module only knows which Clips rows are
+ * involved and which template to render. Share invites are deliberately NOT
+ * routed through the `emailNotifications` preference — they have their own
+ * delivery path.
+ */
+
+import {
+  notifyActivity,
+  runActivityNotification,
+  type ActivityNotificationResult,
+} from "@agent-native/core/server";
+import { filterRecipientsByResourceAccess } from "@agent-native/core/sharing";
+import { and, eq } from "drizzle-orm";
+
+import { CLIPS_USER_PREFS_KEY } from "../../shared/clips-ai-prefs.js";
+import { getDb, schema } from "../db/index.js";
+import { sendClipsTransactionalEmail } from "./transactional-email-templates.js";
+
+/**
+ * `recording-missing` is kept distinct from `no-recipients`: one means the row
+ * we were asked to notify about could not be read, the other means nobody
+ * wanted the email.
+ */
+export type ClipsActivityNotificationResult =
+  | ActivityNotificationResult
+  | { status: "recording-missing"; sent: []; failed: [] };
+
+const RECORDING_MISSING: ClipsActivityNotificationResult = {
+  status: "recording-missing",
+  sent: [],
+  failed: [],
+};
+
+const LOG_LABEL = "[clips] activity notification";
+
+async function getRecording(recordingId: string) {
+  const [row] = await getDb()
+    .select({
+      id: schema.recordings.id,
+      title: schema.recordings.title,
+      ownerEmail: schema.recordings.ownerEmail,
+      orgId: schema.recordings.orgId,
+    })
+    .from(schema.recordings)
+    .where(eq(schema.recordings.id, recordingId))
+    .limit(1);
+  return row ?? null;
+}
+
+async function threadParticipants(
+  recordingId: string,
+  threadId: string,
+): Promise<string[]> {
+  const rows = await getDb()
+    .select({ authorEmail: schema.recordingComments.authorEmail })
+    .from(schema.recordingComments)
+    .where(
+      and(
+        eq(schema.recordingComments.recordingId, recordingId),
+        eq(schema.recordingComments.threadId, threadId),
+      ),
+    );
+  return rows.map((row) => row.authorEmail);
+}
+
+export interface RecordingCommentNotificationInput {
+  recordingId: string;
+  threadId: string;
+  authorEmail: string;
+  authorName?: string | null;
+  content: string;
+  videoTimestampMs?: number | null;
+  isReply?: boolean;
+}
+
+export async function notifyRecordingComment(
+  input: RecordingCommentNotificationInput,
+): Promise<ClipsActivityNotificationResult> {
+  return runActivityNotification(LOG_LABEL, () =>
+    deliverRecordingCommentEmails(input),
+  );
+}
+
+async function deliverRecordingCommentEmails(
+  input: RecordingCommentNotificationInput,
+): Promise<ClipsActivityNotificationResult> {
+  const recording = await getRecording(input.recordingId);
+  if (!recording) {
+    console.error(`${LOG_LABEL}: recording ${input.recordingId} not found`);
+    return RECORDING_MISSING;
+  }
+
+  const candidates = [recording.ownerEmail];
+  if (input.isReply) {
+    candidates.push(
+      ...(await threadParticipants(input.recordingId, input.threadId)),
+    );
+  }
+
+  // Thread rows are history, not an access grant: a viewer whose share was
+  // revoked must stop receiving the recording's comment bodies.
+  const allowed = await filterRecipientsByResourceAccess({
+    resourceType: "recording",
+    resourceId: recording.id,
+    emails: candidates,
+    orgId: recording.orgId,
+  });
+
+  return notifyActivity({
+    candidates: allowed,
+    actorEmail: input.authorEmail,
+    preferenceKey: CLIPS_USER_PREFS_KEY,
+    logLabel: LOG_LABEL,
+    send: (to) =>
+      sendClipsTransactionalEmail({
+        kind: "activity-comment",
+        to,
+        recordingId: recording.id,
+        title: recording.title,
+        authorEmail: input.authorEmail,
+        authorName: input.authorName ?? null,
+        content: input.content,
+        videoTimestampMs: input.videoTimestampMs ?? null,
+        isReply: input.isReply ?? false,
+      }),
+  });
+}
+
+export interface RecordingReactionNotificationInput {
+  recordingId: string;
+  emoji: string;
+  viewerEmail: string;
+  viewerName?: string | null;
+  videoTimestampMs?: number | null;
+  extraRecipients?: (string | null | undefined)[];
+}
+
+export async function notifyRecordingReaction(
+  input: RecordingReactionNotificationInput,
+): Promise<ClipsActivityNotificationResult> {
+  return runActivityNotification(LOG_LABEL, () =>
+    deliverRecordingReactionEmails(input),
+  );
+}
+
+async function deliverRecordingReactionEmails(
+  input: RecordingReactionNotificationInput,
+): Promise<ClipsActivityNotificationResult> {
+  const recording = await getRecording(input.recordingId);
+  if (!recording) {
+    console.error(`${LOG_LABEL}: recording ${input.recordingId} not found`);
+    return RECORDING_MISSING;
+  }
+
+  const allowed = await filterRecipientsByResourceAccess({
+    resourceType: "recording",
+    resourceId: recording.id,
+    emails: [recording.ownerEmail, ...(input.extraRecipients ?? [])].filter(
+      (email): email is string => Boolean(email),
+    ),
+    orgId: recording.orgId,
+  });
+
+  return notifyActivity({
+    candidates: allowed,
+    actorEmail: input.viewerEmail,
+    preferenceKey: CLIPS_USER_PREFS_KEY,
+    logLabel: LOG_LABEL,
+    send: (to) =>
+      sendClipsTransactionalEmail({
+        kind: "activity-reaction",
+        to,
+        recordingId: recording.id,
+        title: recording.title,
+        emoji: input.emoji,
+        authorEmail: input.viewerEmail,
+        authorName: input.viewerName ?? null,
+        videoTimestampMs: input.videoTimestampMs ?? null,
+      }),
+  });
+}

package/corpus/templates/clips/server/lib/recording-media-cleanup.ts

@@ -7,6 +7,7 @@ interface RecordingMediaUrls {
   videoUrl?: string | null;
   thumbnailUrl?: string | null;
   animatedThumbnailUrl?: string | null;
+  filmstripUrl?: string | null;
 }
 
 export interface RecordingMediaCleanupResult {
@@ -25,6 +26,7 @@ export function recordingMediaUrls(recording: RecordingMediaUrls): string[] {
     recording.videoUrl,
     recording.thumbnailUrl,
     recording.animatedThumbnailUrl,
+    recording.filmstripUrl,
   ];
   return [...new Set(urls.filter((url): url is string => Boolean(url)))];
 }