@agent-native/core 0.134.0 → 0.135.0

package/corpus/templates/clips/actions/lib/ensure-recording-filmstrip.ts

@@ -0,0 +1,258 @@
+/**
+ * Generate and attach a timeline filmstrip sprite to a stored recording.
+ *
+ * This is the sibling of `ensure-seekable-video.ts` and follows the same shape:
+ * re-fetch the stored provider media, run one ffmpeg pass, upload the result,
+ * and point the recording row at it. Nothing here is destructive — a recording
+ * without a sprite still renders a filmstrip, because the editor falls back to
+ * extracting frames in the browser.
+ *
+ * Local/dev media (relative `/api/video/...` URLs backed by application_state)
+ * is skipped rather than special-cased: there is no absolute URL to fetch
+ * server-side, and the browser fallback already covers that environment.
+ */
+
+import {
+  readAppState,
+  writeAppState,
+} from "@agent-native/core/application-state";
+import { uploadFile } from "@agent-native/core/file-upload";
+import { and, eq, isNull } from "drizzle-orm";
+
+import { getDb, schema } from "../../server/db/index.js";
+import { deleteRecordingMediaObjects } from "../../server/lib/recording-media-cleanup.js";
+import { ownerEmailMatches } from "../../server/lib/recordings.js";
+import {
+  DEFAULT_FILMSTRIP_FRAME_COUNT,
+  generateFilmstripSprite,
+  type FilmstripSpriteGrid,
+} from "../../server/lib/video-filmstrip-sprite.js";
+import { isLoomRecordingSource } from "../../shared/loom.js";
+import {
+  fetchProviderBytes,
+  isRemoteProviderUrl,
+} from "./ensure-seekable-video.js";
+
+export type EnsureFilmstripStatus =
+  | "generated"
+  | "already-generated"
+  | "not-found"
+  | "skipped-not-ready"
+  | "skipped-no-media"
+  | "skipped-local-media"
+  | "skipped-no-duration"
+  | "skipped-no-ffmpeg"
+  | "skipped-fetch-failed"
+  | "skipped-too-large"
+  | "skipped-upload-failed"
+  | "failed-ffmpeg"
+  | "failed-empty-output";
+
+export interface EnsureFilmstripResult {
+  recordingId: string;
+  status: EnsureFilmstripStatus;
+  changed: boolean;
+  filmstripUrl?: string | null;
+  grid?: FilmstripSpriteGrid;
+  detail?: string;
+}
+
+/** application_state key recording which media a sprite was generated from. */
+export function filmstripMarkerKey(recordingId: string): string {
+  return `recording-filmstrip-${recordingId}`;
+}
+
+async function isAlreadyGenerated(
+  recordingId: string,
+  videoUrl: string,
+): Promise<boolean> {
+  const marker = await readAppState(filmstripMarkerKey(recordingId)).catch(
+    () => null,
+  );
+  return Boolean(
+    marker &&
+    typeof marker === "object" &&
+    (marker as { videoUrl?: unknown }).videoUrl === videoUrl,
+  );
+}
+
+/**
+ * Ensure one recording has a filmstrip sprite. Owner-scoped: pass the resolved
+ * owner email so the lookup can only touch that owner's rows.
+ */
+export async function ensureRecordingFilmstrip(params: {
+  recordingId: string;
+  ownerEmail: string;
+  frameCount?: number;
+  force?: boolean;
+}): Promise<EnsureFilmstripResult> {
+  const {
+    recordingId,
+    ownerEmail,
+    frameCount = DEFAULT_FILMSTRIP_FRAME_COUNT,
+    force = false,
+  } = params;
+  const db = getDb();
+
+  const [rec] = await db
+    .select({
+      id: schema.recordings.id,
+      status: schema.recordings.status,
+      videoUrl: schema.recordings.videoUrl,
+      durationMs: schema.recordings.durationMs,
+      filmstripUrl: schema.recordings.filmstripUrl,
+      sourceAppName: schema.recordings.sourceAppName,
+    })
+    .from(schema.recordings)
+    .where(
+      and(
+        eq(schema.recordings.id, recordingId),
+        ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+      ),
+    );
+
+  if (!rec) return { recordingId, status: "not-found", changed: false };
+  if (rec.status !== "ready") {
+    return { recordingId, status: "skipped-not-ready", changed: false };
+  }
+  if (isLoomRecordingSource(rec)) {
+    return { recordingId, status: "skipped-local-media", changed: false };
+  }
+  if (!rec.videoUrl) {
+    return { recordingId, status: "skipped-no-media", changed: false };
+  }
+  if (!isRemoteProviderUrl(rec.videoUrl)) {
+    return {
+      recordingId,
+      status: "skipped-local-media",
+      changed: false,
+      filmstripUrl: rec.filmstripUrl,
+    };
+  }
+  if (rec.durationMs <= 0) {
+    return { recordingId, status: "skipped-no-duration", changed: false };
+  }
+
+  if (
+    !force &&
+    rec.filmstripUrl &&
+    (await isAlreadyGenerated(recordingId, rec.videoUrl))
+  ) {
+    return {
+      recordingId,
+      status: "already-generated",
+      changed: false,
+      filmstripUrl: rec.filmstripUrl,
+    };
+  }
+
+  const fetched = await fetchProviderBytes(rec.videoUrl);
+  if (!fetched.ok) {
+    return { recordingId, status: fetched.reason, changed: false };
+  }
+
+  const sprite = await generateFilmstripSprite({
+    mediaBytes: fetched.bytes,
+    durationMs: rec.durationMs,
+    frameCount,
+  });
+
+  if (sprite.status !== "generated" || !sprite.sprite) {
+    return {
+      recordingId,
+      status: sprite.status,
+      changed: false,
+      filmstripUrl: rec.filmstripUrl,
+      detail: sprite.detail,
+    };
+  }
+
+  const upload = await uploadFile({
+    data: sprite.sprite.bytes,
+    filename: `${recordingId}-filmstrip-${Date.now()}.jpg`,
+    mimeType: "image/jpeg",
+    ownerEmail,
+    stableUrl: true,
+    recordAsset: false,
+  }).catch((err) => {
+    console.warn("[ensure-recording-filmstrip] sprite upload failed", {
+      recordingId,
+      error: err instanceof Error ? err.message : String(err),
+    });
+    return null;
+  });
+
+  if (!upload?.url) {
+    return {
+      recordingId,
+      status: "skipped-upload-failed",
+      changed: false,
+      filmstripUrl: rec.filmstripUrl,
+    };
+  }
+
+  const { grid } = sprite.sprite;
+  const updated = await db
+    .update(schema.recordings)
+    .set({
+      filmstripUrl: upload.url,
+      filmstripFrameCount: grid.frameCount,
+      filmstripColumns: grid.columns,
+      filmstripRows: grid.rows,
+      filmstripFrameWidth: grid.frameWidth,
+      filmstripFrameHeight: grid.frameHeight,
+      updatedAt: new Date().toISOString(),
+    })
+    .where(
+      and(
+        eq(schema.recordings.id, recordingId),
+        ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+        // Don't attach a sprite generated from media the row no longer points at.
+        eq(schema.recordings.videoUrl, rec.videoUrl),
+        // Ensure no concurrent job modified filmstripUrl in the interim.
+        rec.filmstripUrl
+          ? eq(schema.recordings.filmstripUrl, rec.filmstripUrl)
+          : isNull(schema.recordings.filmstripUrl),
+      ),
+    )
+    .returning({
+      id: schema.recordings.id,
+      filmstripUrl: schema.recordings.filmstripUrl,
+    });
+
+  if (updated.length !== 1 || updated[0]?.filmstripUrl !== upload.url) {
+    await deleteRecordingMediaObjects({
+      id: recordingId,
+      filmstripUrl: upload.url,
+    }).catch(() => {});
+    return {
+      recordingId,
+      status: "skipped-upload-failed",
+      changed: false,
+      filmstripUrl: rec.filmstripUrl,
+      detail: "Recording changed while the filmstrip sprite was uploading.",
+    };
+  }
+
+  if (rec.filmstripUrl && rec.filmstripUrl !== upload.url) {
+    await deleteRecordingMediaObjects({
+      id: recordingId,
+      filmstripUrl: rec.filmstripUrl,
+    }).catch(() => {});
+  }
+
+  await writeAppState(filmstripMarkerKey(recordingId), {
+    recordingId,
+    videoUrl: rec.videoUrl,
+    at: new Date().toISOString(),
+  });
+  await writeAppState("refresh-signal", { ts: Date.now() });
+
+  return {
+    recordingId,
+    status: "generated",
+    changed: true,
+    filmstripUrl: upload.url,
+    grid,
+  };
+}

package/corpus/templates/clips/actions/lib/ensure-seekable-video.ts

@@ -95,14 +95,22 @@ async function isAlreadyMarked(
   );
 }
 
-function isRemoteProviderUrl(videoUrl: string | null | undefined): boolean {
+export function isRemoteProviderUrl(
+  videoUrl: string | null | undefined,
+): boolean {
   return typeof videoUrl === "string" && /^https:\/\//i.test(videoUrl.trim());
 }
 
-async function fetchProviderBytes(
+/**
+ * Download stored provider media for server-side processing. Shared with the
+ * filmstrip sprite path — both need the same size cap and timeout, and both
+ * report the same two failure reasons rather than an empty buffer.
+ */
+export async function fetchProviderBytes(
   videoUrl: string,
 ): Promise<
-  { ok: true; bytes: Uint8Array } | { ok: false; reason: EnsureSeekableStatus }
+  | { ok: true; bytes: Uint8Array }
+  | { ok: false; reason: "skipped-fetch-failed" | "skipped-too-large" }
 > {
   const controller = new AbortController();
   const timeout = setTimeout(

package/corpus/templates/clips/actions/react-to-comment.ts

@@ -17,6 +17,7 @@ import { and, eq } from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
+import { notifyRecordingReaction } from "../server/lib/activity-notifications.js";
 
 const MAX_CAS_ATTEMPTS = 3;
 
@@ -90,12 +91,24 @@ export default defineAction({
         .returning({ id: schema.recordingComments.id });
 
       if (updated.length > 0) {
+        // Removing a reaction is not an event worth emailing about.
+        const notified = had
+          ? null
+          : await notifyRecordingReaction({
+              recordingId: existing.recordingId,
+              emoji: args.emoji,
+              viewerEmail,
+              videoTimestampMs: existing.videoTimestampMs,
+              extraRecipients: [existing.authorEmail],
+            });
+
         await writeAppState("refresh-signal", { ts: Date.now() });
         return {
           id: args.commentId,
           emoji: args.emoji,
           reacted: !had,
           reactions: next,
+          notified,
         };
       }
     }

package/corpus/templates/clips/actions/react-to-recording.ts

@@ -12,6 +12,7 @@ import { assertAccess } from "@agent-native/core/sharing";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
+import { notifyRecordingReaction } from "../server/lib/activity-notifications.js";
 import { nanoid } from "../server/lib/recordings.js";
 
 export default defineAction({
@@ -50,8 +51,16 @@ export default defineAction({
       createdAt: now,
     });
 
+    const notified = await notifyRecordingReaction({
+      recordingId: args.recordingId,
+      emoji: args.emoji,
+      viewerEmail,
+      viewerName: args.viewerName,
+      videoTimestampMs: args.videoTimestampMs,
+    });
+
     await writeAppState("refresh-signal", { ts: Date.now() });
 
-    return { id };
+    return { id, notified };
   },
 });

package/corpus/templates/clips/actions/reply-to-comment.ts

@@ -15,6 +15,7 @@ import { eq } from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
+import { notifyRecordingComment } from "../server/lib/activity-notifications.js";
 import { nanoid } from "../server/lib/recordings.js";
 
 export default defineAction({
@@ -58,12 +59,22 @@ export default defineAction({
       updatedAt: now,
     });
 
+    const notified = await notifyRecordingComment({
+      recordingId: parent.recordingId,
+      threadId: parent.threadId,
+      authorEmail,
+      authorName: args.authorName,
+      content: args.content,
+      videoTimestampMs: parent.videoTimestampMs,
+      isReply: true,
+    });
+
     await writeAppState("refresh-signal", { ts: Date.now() });
 
     console.log(
       `Replied to comment ${args.commentId} (thread: ${parent.threadId})`,
     );
 
-    return { id, threadId: parent.threadId, parentId: parent.id };
+    return { id, threadId: parent.threadId, parentId: parent.id, notified };
   },
 });

package/corpus/templates/clips/actions/stitch-recordings.ts

@@ -36,7 +36,7 @@ import { parseEdits, serializeEdits } from "../app/lib/timestamp-mapping.js";
 import { getDb, schema } from "../server/db/index.js";
 import {
   getCurrentOwnerEmail,
-  getOrganizationDefaultVisibility,
+  getDefaultRecordingVisibility,
   nanoid,
   ownerEmailMatches,
 } from "../server/lib/recordings.js";
@@ -121,7 +121,7 @@ export default defineAction({
     }
     const organizationId = ordered[0].organizationId;
     const defaultVisibility =
-      await getOrganizationDefaultVisibility(organizationId);
+      await getDefaultRecordingVisibility(organizationId);
 
     const totalDuration =
       args.durationMs ??

package/corpus/templates/clips/app/components/editor/editor-layout.tsx

@@ -58,6 +58,11 @@ import {
   type EditsJson,
 } from "@/lib/timestamp-mapping";
 import { cn } from "@/lib/utils";
+import {
+  extractFilmstripThumbnails,
+  type FilmstripFrame,
+  type FilmstripSprite,
+} from "@/lib/video-filmstrip";
 import { computePeaks, type WaveformPeaks } from "@/lib/waveform-peaks";
 
 import { ChaptersEditor } from "./chapters-editor";
@@ -76,7 +81,7 @@ export interface EditorLayoutProps {
   className?: string;
 }
 
-const WAVEFORM_HEIGHT = 120;
+const WAVEFORM_HEIGHT = 100;
 const MIN_TIMELINE_ZOOM = 1;
 const MAX_TIMELINE_ZOOM = 50;
 
@@ -244,10 +249,20 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
     return () => ro.disconnect();
   }, []);
 
-  const totalWidth = Math.max(
-    viewportWidth,
-    Math.floor(viewportWidth * Math.max(1, zoom)),
-  );
+  const baseTrackWidth = useMemo(() => {
+    if (durationMs <= 0 || viewportWidth <= 0) return viewportWidth;
+    const durationSec = durationMs / 1000;
+    // Scale timeline track width proportionally with duration so long videos scale & scroll (bounded to safe max allocation)
+    const durationScaledPx = Math.min(8192, Math.round(durationSec * 14));
+    return Math.max(viewportWidth, durationScaledPx);
+  }, [durationMs, viewportWidth]);
+
+  const totalWidth = useMemo(() => {
+    return Math.max(
+      baseTrackWidth,
+      Math.floor(baseTrackWidth * Math.max(1, zoom)),
+    );
+  }, [baseTrackWidth, zoom]);
 
   const calculateAnchoredScrollLeft = useCallback(
     (
@@ -255,8 +270,8 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
       anchor?: { anchorRatio?: number; viewportX?: number },
     ) => {
       const nextTotalWidth = Math.max(
-        viewportWidth,
-        Math.floor(viewportWidth * Math.max(1, nextZoom)),
+        baseTrackWidth,
+        Math.floor(baseTrackWidth * Math.max(1, nextZoom)),
       );
       const maxScrollLeft = Math.max(0, nextTotalWidth - viewportWidth);
       const anchorMs = selectionRange
@@ -277,7 +292,7 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
       const anchorX = anchorRatio * nextTotalWidth;
       return Math.max(0, Math.min(maxScrollLeft, anchorX - viewportX));
     },
-    [durationMs, playheadMs, selectionRange, viewportWidth],
+    [baseTrackWidth, durationMs, playheadMs, selectionRange, viewportWidth],
   );
 
   const setAnchoredZoom = useCallback(
@@ -319,8 +334,8 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
       viewportX: number,
     ) => {
       const sourceTotalWidth = Math.max(
-        viewportWidth,
-        Math.floor(viewportWidth * Math.max(1, sourceZoom)),
+        baseTrackWidth,
+        Math.floor(baseTrackWidth * Math.max(1, sourceZoom)),
       );
       return Math.max(
         0,
@@ -390,7 +405,7 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
       el.removeEventListener("gesturechange", handleGestureChange);
       el.removeEventListener("gestureend", handleGestureEnd);
     };
-  }, [scrollLeft, setAnchoredZoom, viewportWidth, zoom]);
+  }, [baseTrackWidth, scrollLeft, setAnchoredZoom, viewportWidth, zoom]);
 
   // Sync the <video> to play state.
   useEffect(() => {
@@ -498,6 +513,96 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
     };
   }, [recordingId, waveformMediaUrl]);
 
+  // Filmstrip drawn behind the waveform. A server-generated sprite is one
+  // cached image request and is preferred; browser extraction is the fallback
+  // for hosts without ffmpeg and for local/dev media the server can't fetch.
+  const filmstripSprite = useMemo<FilmstripSprite | null>(() => {
+    const url = recording?.filmstripUrl;
+    const frameCount = Number(recording?.filmstripFrameCount ?? 0);
+    const columns = Number(recording?.filmstripColumns ?? 0);
+    if (!url || frameCount <= 0 || columns <= 0) return null;
+    return {
+      url,
+      frameCount,
+      columns,
+      rows: Number(recording?.filmstripRows ?? 1) || 1,
+      frameWidth: Number(recording?.filmstripFrameWidth ?? 0) || 160,
+      frameHeight: Number(recording?.filmstripFrameHeight ?? 0) || 90,
+    };
+  }, [
+    recording?.filmstripUrl,
+    recording?.filmstripFrameCount,
+    recording?.filmstripColumns,
+    recording?.filmstripRows,
+    recording?.filmstripFrameWidth,
+    recording?.filmstripFrameHeight,
+  ]);
+
+  const [filmstripFrames, setFilmstripFrames] = useState<FilmstripFrame[]>([]);
+
+  useEffect(() => {
+    setFilmstripFrames([]);
+  }, [recordingId]);
+
+  // Cells should read as video frames, so aim for one per `height * aspect` of
+  // track. Bucketed so ordinary window resizing does not re-extract, and based
+  // on the unzoomed width — a zoomed fallback strip stretches, which is one of
+  // the reasons the server sprite is the preferred path.
+  const filmstripFrameCount = useMemo(() => {
+    const bucketedWidth = Math.max(240, Math.round(viewportWidth / 120) * 120);
+    const cellWidth = WAVEFORM_HEIGHT * (16 / 9);
+    return Math.min(48, Math.max(6, Math.round(bucketedWidth / cellWidth)));
+  }, [viewportWidth]);
+
+  useEffect(() => {
+    // A sprite already covers the whole clip — don't decode the video again.
+    if (filmstripSprite) {
+      setFilmstripFrames([]);
+      return;
+    }
+    if (!waveformMediaUrl || durationMs <= 0) {
+      setFilmstripFrames([]);
+      return;
+    }
+    setFilmstripFrames([]);
+    let cancelled = false;
+    // `waveformMediaUrl`, not `videoUrl`: reading frames back out of a
+    // cross-origin video taints the canvas, so provider media must come
+    // through the same-origin proxy first.
+    extractFilmstripThumbnails({
+      videoUrl: waveformMediaUrl,
+      durationMs,
+      frameCount: filmstripFrameCount,
+    })
+      .then((result) => {
+        if (cancelled) return;
+        if (result.status !== "ok") {
+          console.warn("[editor] filmstrip extraction failed", {
+            recordingId,
+            status: result.status,
+            detail: result.detail,
+          });
+        }
+        setFilmstripFrames(result.frames);
+      })
+      .catch((err) => {
+        if (cancelled) return;
+        console.warn("[editor] filmstrip extraction threw", {
+          recordingId,
+          error: err instanceof Error ? err.message : String(err),
+        });
+      });
+    return () => {
+      cancelled = true;
+    };
+  }, [
+    recordingId,
+    waveformMediaUrl,
+    durationMs,
+    filmstripFrameCount,
+    filmstripSprite,
+  ]);
+
   // --- actions ------------------------------------------------------------
   const trim = useActionMutation("trim-recording");
   const split = useActionMutation("split-recording");
@@ -550,12 +655,22 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
       ) {
         e.preventDefault();
         undo.mutate({ recordingId });
-      } else if (e.key.toLowerCase() === "i") {
+      } else if (
+        !e.ctrlKey &&
+        !e.metaKey &&
+        !e.altKey &&
+        e.key.toLowerCase() === "i"
+      ) {
         setSelectionRange((r) => ({
           startMs: playheadMs,
           endMs: r?.endMs && r.endMs > playheadMs ? r.endMs : playheadMs + 1000,
         }));
-      } else if (e.key.toLowerCase() === "o") {
+      } else if (
+        !e.ctrlKey &&
+        !e.metaKey &&
+        !e.altKey &&
+        e.key.toLowerCase() === "o"
+      ) {
         setSelectionRange((r) => ({
           startMs:
             r?.startMs && r.startMs < playheadMs
@@ -563,7 +678,12 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
               : Math.max(0, playheadMs - 1000),
           endMs: playheadMs,
         }));
-      } else if (e.key.toLowerCase() === "x") {
+      } else if (
+        !e.ctrlKey &&
+        !e.metaKey &&
+        !e.altKey &&
+        e.key.toLowerCase() === "x"
+      ) {
         // Cut: trim the current selection range
         const range = selectionRange;
         if (range) {
@@ -582,7 +702,12 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
               toast.error(err?.message ?? t("editorLayout.cutFailed")),
             );
         }
-      } else if (e.key.toLowerCase() === "s") {
+      } else if (
+        !e.ctrlKey &&
+        !e.metaKey &&
+        !e.altKey &&
+        e.key.toLowerCase() === "s"
+      ) {
         // Split at playhead
         e.preventDefault();
         split
@@ -693,6 +818,8 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
             <div className="relative min-w-0 overflow-hidden">
               <Waveform
                 peaks={peaks}
+                sprite={filmstripSprite}
+                frames={filmstripFrames}
                 width={viewportWidth}
                 height={WAVEFORM_HEIGHT}
                 zoom={zoom}
@@ -717,14 +844,16 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) {
                     transform: `translateX(${-scrollLeft}px)`,
                   }}
                 >
-                  <TrimHandles
-                    width={totalWidth}
-                    height={WAVEFORM_HEIGHT}
-                    value={effectiveSelection}
-                    onChange={setSelectionRange}
-                    durationMs={durationMs}
-                    splitPoints={splitPoints}
-                  />
+                  {durationMs > 0 && (
+                    <TrimHandles
+                      width={totalWidth}
+                      height={WAVEFORM_HEIGHT}
+                      value={effectiveSelection}
+                      onChange={setSelectionRange}
+                      durationMs={durationMs}
+                      splitPoints={splitPoints}
+                    />
+                  )}
                 </div>
               </div>
             </div>