@agent-native/core 0.92.12 → 0.94.0

package/corpus/templates/clips/actions/request-transcript.ts

@@ -9,7 +9,8 @@
  *
  * Cloud fallback provider selection:
  *   1. Builder.io transcription (Gemini 3.1 Flash-Lite behind the Builder
- *      proxy) when Builder is connected.
+ *      proxy) when Builder is connected; if that model is unavailable in the
+ *      deployment region, retry the Builder gateway's default model.
  *   2. `GROQ_API_KEY` → Groq's fast speech-to-text fallback.
  *   3. Neither → keep any native transcript or fail with a clear reason.
  *
@@ -51,6 +52,7 @@ import { and, eq } from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
+import { dispatchPostFinalizeJob } from "../server/lib/post-finalize-dispatch.js";
 import {
   getCurrentOwnerEmail,
   ownerEmailMatches,
@@ -87,6 +89,7 @@ import {
   normalizeProviderTranscript,
 } from "./lib/provider-transcript.js";
 import { isAutoTitleReplaceable } from "./lib/title-source.js";
+import regenerateSummary from "./regenerate-summary.js";
 import regenerateTitle from "./regenerate-title.js";
 
 interface SpeechToTextSegment {
@@ -135,6 +138,49 @@ const MEDIA_FETCH_BASE_TIMEOUT_MS = 30_000;
 const MEDIA_FETCH_PER_50MB_MS = 10_000;
 const ESTIMATED_VIDEO_BYTES_PER_MINUTE = 5 * 1024 * 1024;
 
+function builderErrorText(error: unknown): string {
+  if (error instanceof Error) {
+    const cause = (error as Error & { cause?: unknown }).cause;
+    return `${error.message} ${cause ? builderErrorText(cause) : ""}`.trim();
+  }
+  return String(error);
+}
+
+export function isBuilderModelAvailabilityError(error: unknown): boolean {
+  const message = builderErrorText(error).toLowerCase();
+  const identifiesModel =
+    message.includes("model") || message.includes("gemini");
+  const identifiesAvailability =
+    message.includes("not available") ||
+    message.includes("unavailable") ||
+    message.includes("unsupported") ||
+    message.includes("not supported") ||
+    message.includes("not found") ||
+    message.includes("region") ||
+    message.includes("location");
+  return identifiesModel && identifiesAvailability;
+}
+
+export async function transcribeWithBuilderModelFallback(
+  options: Omit<Parameters<typeof transcribeWithBuilder>[0], "model">,
+) {
+  try {
+    return await transcribeWithBuilder({
+      ...options,
+      model: BUILDER_GEMINI_TRANSCRIPTION_MODEL,
+    });
+  } catch (error) {
+    if (!isBuilderModelAvailabilityError(error)) throw error;
+    console.warn(
+      `[clips] Builder transcription model ${BUILDER_GEMINI_TRANSCRIPTION_MODEL} is unavailable; retrying the gateway default model.`,
+    );
+    // `model` is optional on the Builder transcription endpoint. Omitting it
+    // restores the gateway's region-aware default that Clips used before the
+    // explicit Gemini model was introduced.
+    return transcribeWithBuilder(options);
+  }
+}
+
 function clampTimeoutMs(value: number): number {
   return Math.max(
     BUILDER_TRANSCRIPTION_MIN_TIMEOUT_MS,
@@ -195,20 +241,9 @@ export function recordingMediaFetchTimeoutMs(
 
 // Bounded automatic retry for transient failures (ffmpeg timeout, transient
 // provider network/5xx errors) — NOT for permanent failures like "no audio
-// track" or a missing/rejected API key. Mirrors the fire-and-forget
-// background pattern already used for post-finalize transcription and
-// seekable remux in finalize-recording.ts, just with a delay before the
-// retry instead of running immediately.
-//
-// Serverless caveat: Clips deploys to Netlify Functions (NITRO_PRESET=netlify),
-// where a `setTimeout` scheduled after the handler returns can be frozen along
-// with the rest of the sandbox before it fires. This best-effort retry still
-// helps on any host that keeps the process warm (local/self-hosted, or a
-// Lambda instance reused for a later request), and it is never worse than
-// today's behavior — a clip that doesn't get an automatic retry just settles
-// into "failed", exactly as it does now, and stays retryable via
-// `request-transcript` (UI retry button or agent). Keeping the backoff short
-// maximizes the chance it fires before the sandbox freezes.
+// track" or a missing/rejected API key. Each retry is self-dispatched into a
+// fresh request so serverless runtimes cannot freeze a timer left behind by
+// the completed transcription request.
 const MAX_AUTO_TRANSCRIPT_RETRIES = 2;
 const AUTO_TRANSCRIPT_RETRY_BACKOFF_MS = [5_000, 20_000];
 
@@ -237,9 +272,7 @@ function isTransientTranscriptionError(err: unknown): boolean {
 
 /**
  * Schedule a bounded, backed-off automatic retry of `request-transcript` for
- * a transient failure. Fire-and-forget in the Nitro process — same mechanism
- * `finalize-recording.ts` uses to kick off transcription/remux passes in the
- * background, just delayed instead of immediate.
+ * a transient failure in a fresh server request.
  *
  * `nextRetryCount` must already be persisted to `recording_transcripts` by the
  * caller BEFORE this is invoked (not inside the timer) so the retry budget
@@ -262,20 +295,17 @@ function scheduleAutoTranscriptRetry({
     AUTO_TRANSCRIPT_RETRY_BACKOFF_MS[
       AUTO_TRANSCRIPT_RETRY_BACKOFF_MS.length - 1
     ];
-  setTimeout(() => {
-    void Promise.resolve(
-      requestTranscriptAction.run({
-        recordingId,
-        force: true,
-        retryAttempt: nextRetryCount,
-      }),
-    ).catch((err: unknown) => {
-      console.warn(
-        `[clips] auto-retry transcription failed for ${recordingId} (attempt ${nextRetryCount}):`,
-        (err as Error)?.message ?? String(err),
-      );
-    });
-  }, backoffMs);
+  void dispatchPostFinalizeJob({
+    recordingId,
+    kind: "transcript",
+    delayMs: backoffMs,
+    retryAttempt: nextRetryCount,
+  }).catch((err: unknown) => {
+    console.warn(
+      `[clips] auto-retry transcription dispatch failed for ${recordingId} (attempt ${nextRetryCount}):`,
+      (err as Error)?.message ?? String(err),
+    );
+  });
 }
 
 function queueBrainExport(recordingId: string): void {
@@ -728,6 +758,39 @@ async function cleanupNativeTranscript({
   }
 }
 
+async function generateRecordingMetadata({
+  recordingId,
+  title,
+  titleSource,
+  description,
+  transcriptText,
+}: {
+  recordingId: string;
+  title: string | null | undefined;
+  titleSource: string | null | undefined;
+  description: string | null | undefined;
+  transcriptText: string;
+}): Promise<{ titleQueued: boolean; summaryQueued: boolean }> {
+  if (isAutoTitleReplaceable(title, titleSource)) {
+    await regenerateTitle.run({
+      recordingId,
+      transcriptText,
+      includeSummary: !description?.trim(),
+    });
+    return {
+      titleQueued: true,
+      summaryQueued: !description?.trim(),
+    };
+  }
+
+  if (!description?.trim()) {
+    await regenerateSummary.run({ recordingId });
+    return { titleQueued: false, summaryQueued: true };
+  }
+
+  return { titleQueued: false, summaryQueued: false };
+}
+
 async function completeReadyTranscript({
   db,
   recordingId,
@@ -749,12 +812,14 @@ async function completeReadyTranscript({
   provider: "existing" | "native";
   cleanupQueued: boolean;
   titleQueued: boolean;
+  summaryQueued: boolean;
   preserved?: true;
 }> {
   const [recForTitle] = await db
     .select({
       title: schema.recordings.title,
       titleSource: schema.recordings.titleSource,
+      description: schema.recordings.description,
       durationMs: schema.recordings.durationMs,
     })
     .from(schema.recordings)
@@ -789,7 +854,7 @@ async function completeReadyTranscript({
     }
   }
 
-  void cleanupNativeTranscript({
+  const cleanupPromise = cleanupNativeTranscript({
     db,
     recordingId,
     ownerEmail,
@@ -800,24 +865,47 @@ async function completeReadyTranscript({
       `[clips] native transcript cleanup failed for ${recordingId}:`,
       (err as Error)?.message ?? String(err),
     );
+    return { cleaned: false };
   });
 
-  const titleQueued = !!(
-    recForTitle &&
-    isAutoTitleReplaceable(recForTitle.title, recForTitle.titleSource)
-  );
-  if (titleQueued) {
-    void Promise.resolve(
-      regenerateTitle.run({
+  const metadataPromise = recForTitle
+    ? generateRecordingMetadata({
         recordingId,
+        title: recForTitle.title,
+        titleSource: recForTitle.titleSource,
+        description: recForTitle.description,
         transcriptText: fullText,
-      }),
-    ).catch((err: unknown) => {
-      console.warn(
-        `[clips] native-transcript title generation failed for ${recordingId}:`,
-        (err as Error)?.message ?? String(err),
-      );
-    });
+      }).catch((err: unknown) => {
+        console.warn(
+          `[clips] native-transcript metadata generation failed for ${recordingId}:`,
+          (err as Error)?.message ?? String(err),
+        );
+        return { titleQueued: false, summaryQueued: false };
+      })
+    : Promise.resolve({ titleQueued: false, summaryQueued: false });
+
+  // Both calls are independent. Await them together so the durable worker stays
+  // alive without serially stacking two model-call timeouts.
+  const [cleanupResult, metadataResult] = await Promise.all([
+    cleanupPromise,
+    metadataPromise,
+  ]);
+
+  if (!recForTitle) {
+    console.warn(
+      `[clips] recording metadata generation skipped because ${recordingId} was not found`,
+    );
+  }
+
+  if (metadataResult.titleQueued) {
+    console.log(
+      `[clips] transcript-backed title generation completed or queued for ${recordingId}`,
+    );
+  }
+  if (metadataResult.summaryQueued) {
+    console.log(
+      `[clips] transcript-backed summary generation queued for ${recordingId}`,
+    );
   }
 
   // Wake the player polling so it picks up the queued cleanup state row
@@ -829,10 +917,11 @@ async function completeReadyTranscript({
   return {
     recordingId,
     status: "ready",
-    cleaned: false,
+    cleaned: cleanupResult.cleaned,
     provider: segmentsJson && segmentsJson !== "[]" ? "existing" : "native",
-    cleanupQueued: true,
-    titleQueued,
+    cleanupQueued: false,
+    titleQueued: metadataResult.titleQueued,
+    summaryQueued: metadataResult.summaryQueued,
     ...(preserved ? { preserved: true as const } : {}),
   };
 }
@@ -854,6 +943,7 @@ async function preserveReadyTranscriptIfAvailable({
   provider: "existing" | "native";
   cleanupQueued: boolean;
   titleQueued: boolean;
+  summaryQueued: boolean;
   preserved?: true;
 } | null> {
   const [current] = await db
@@ -1161,10 +1251,9 @@ const requestTranscriptAction = defineAction({
 
       try {
         const startedAt = Date.now();
-        const builderResult = await transcribeWithBuilder({
+        const builderResult = await transcribeWithBuilderModelFallback({
           audioBytes: audioMedia.audioBytes,
           mimeType: audioMedia.mimeType,
-          model: BUILDER_GEMINI_TRANSCRIPTION_MODEL,
           diarize: false,
           instructions: SPEECH_ONLY_TRANSCRIPTION_INSTRUCTIONS,
           timeoutMs: builderTranscriptionTimeoutMs(rec.durationMs),
@@ -1223,16 +1312,23 @@ const requestTranscriptAction = defineAction({
           .select({
             title: schema.recordings.title,
             titleSource: schema.recordings.titleSource,
+            description: schema.recordings.description,
           })
           .from(schema.recordings)
           .where(eq(schema.recordings.id, args.recordingId))
           .limit(1);
-        if (isAutoTitleReplaceable(freshRec?.title, freshRec?.titleSource)) {
+        if (freshRec) {
           try {
-            await regenerateTitle.run({ recordingId: args.recordingId });
+            await generateRecordingMetadata({
+              recordingId: args.recordingId,
+              title: freshRec.title,
+              titleSource: freshRec.titleSource,
+              description: freshRec.description,
+              transcriptText: fullText,
+            });
           } catch (delegateErr) {
             console.warn(
-              `[clips] auto-title delegation failed for ${args.recordingId}:`,
+              `[clips] automatic metadata generation failed for ${args.recordingId}:`,
               (delegateErr as Error).message,
             );
           }
@@ -1342,6 +1438,7 @@ const requestTranscriptAction = defineAction({
         durationMs: schema.recordings.durationMs,
         title: schema.recordings.title,
         titleSource: schema.recordings.titleSource,
+        description: schema.recordings.description,
       })
       .from(schema.recordings)
       .where(eq(schema.recordings.id, args.recordingId))
@@ -1470,23 +1567,22 @@ const requestTranscriptAction = defineAction({
       await writeAppState("refresh-signal", { ts: Date.now() });
       queueBrainExport(args.recordingId);
 
-      // Auto-title. The clip was just born with the default title and we now
-      // have a transcript to reason over. `regenerate-title` tries the fast
-      // media-pipeline path and only queues an agent fallback when appropriate,
-      // so Builder credit pauses stay paused instead of spawning another AI job.
-      // We intentionally skip this when the user (or agent) has already renamed
-      // the clip so we never clobber a human-authored title.
-      if (isAutoTitleReplaceable(rec.title, rec.titleSource)) {
-        try {
-          await regenerateTitle.run({ recordingId: args.recordingId });
-        } catch (delegateErr) {
-          // Non-fatal — a missing delegation just means the clip keeps its
-          // placeholder title until the user asks the agent to rename it.
-          console.warn(
-            `[clips] auto-title delegation failed for ${args.recordingId}:`,
-            (delegateErr as Error).message,
-          );
-        }
+      // Generate transcript-backed metadata without replacing a human title or
+      // description. The title action keeps any local heuristic replaceable
+      // while its agent refinement runs.
+      try {
+        await generateRecordingMetadata({
+          recordingId: args.recordingId,
+          title: rec.title,
+          titleSource: rec.titleSource,
+          description: rec.description,
+          transcriptText: fullText,
+        });
+      } catch (delegateErr) {
+        console.warn(
+          `[clips] automatic metadata generation failed for ${args.recordingId}:`,
+          (delegateErr as Error).message,
+        );
       }
 
       const elapsedMs = Date.now() - startedAt;

package/corpus/templates/clips/actions/save-browser-transcript.ts

@@ -20,11 +20,11 @@ import { eq } from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
+import { dispatchPostFinalizeJob } from "../server/lib/post-finalize-dispatch.js";
 import { getCurrentOwnerEmail } from "../server/lib/recordings.js";
 import { buildCaptionSegmentsFromText } from "../shared/transcript-segments.js";
 import { booleanParam } from "./lib/cli-params.js";
 import { isAutoTitleReplaceable } from "./lib/title-source.js";
-import regenerateTitle from "./regenerate-title.js";
 
 function nativeSegmentsJson(fullText: string): string {
   return JSON.stringify(buildCaptionSegmentsFromText(fullText));
@@ -218,6 +218,8 @@ export default defineAction({
       .select({
         title: schema.recordings.title,
         titleSource: schema.recordings.titleSource,
+        description: schema.recordings.description,
+        status: schema.recordings.status,
       })
       .from(schema.recordings)
       .where(eq(schema.recordings.id, args.recordingId))
@@ -226,15 +228,14 @@ export default defineAction({
     const titleQueued = !!(
       rec && isAutoTitleReplaceable(rec.title, rec.titleSource)
     );
-    if (titleQueued) {
-      void Promise.resolve(
-        regenerateTitle.run({
-          recordingId: args.recordingId,
-          transcriptText: fullText,
-        }),
-      ).catch((err: unknown) => {
+    const summaryQueued = Boolean(rec && !rec.description?.trim());
+    if (rec?.status === "ready" && (titleQueued || summaryQueued)) {
+      await dispatchPostFinalizeJob({
+        recordingId: args.recordingId,
+        kind: "transcript",
+      }).catch((err: unknown) => {
         console.warn(
-          `[clips] native transcript title generation skipped for ${args.recordingId}:`,
+          `[clips] native transcript metadata dispatch failed for ${args.recordingId}:`,
           (err as Error)?.message ?? String(err),
         );
       });
@@ -246,6 +247,7 @@ export default defineAction({
       provider: args.source ?? "web-speech",
       chars: fullText.length,
       titleQueued,
+      summaryQueued,
     };
   },
 });

package/corpus/templates/clips/actions/view-screen.ts

@@ -3,7 +3,7 @@
  *
  * Reads `navigation` application state and fetches the relevant context
  * (recording + transcript + comments if viewing a recording, folder contents
- * if on library, space list if on spaces, etc.). Returns a single JSON
+ * if on library/shared-with-me, space list if on spaces, etc.). Returns a single JSON
  * snapshot the agent can reason over.
  *
  * Usage:
@@ -17,7 +17,18 @@ import {
 } from "@agent-native/core/application-state";
 import { getRequestUserEmail } from "@agent-native/core/server/request-context";
 import { accessFilter, resolveAccess } from "@agent-native/core/sharing";
-import { and, asc, desc, eq, gte, isNotNull, isNull, lte } from "drizzle-orm";
+import {
+  and,
+  asc,
+  desc,
+  eq,
+  gte,
+  isNotNull,
+  isNull,
+  lte,
+  not,
+  notInArray,
+} from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
@@ -158,13 +169,40 @@ async function fetchBugReportSummary(recordingId: string) {
   };
 }
 
-async function fetchLibrary(folderId?: string) {
+async function fetchLibrary({
+  folderId,
+  shared = false,
+  organizationId,
+}: {
+  folderId?: string;
+  shared?: boolean;
+  organizationId?: string | null;
+}) {
   const db = getDb();
+  const ownerEmail = getRequestUserEmail();
+  if (!ownerEmail) return [];
   const conditions = [
     accessFilter(schema.recordings, schema.recordingShares),
     isNull(schema.recordings.archivedAt),
     isNull(schema.recordings.trashedAt),
   ];
+  if (shared) {
+    conditions.push(
+      not(ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail)),
+    );
+  } else {
+    conditions.push(
+      ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+    );
+    if (organizationId) {
+      conditions.push(eq(schema.recordings.organizationId, organizationId));
+    }
+  }
+  const meetingRecordingIds = db
+    .select({ id: schema.meetings.recordingId })
+    .from(schema.meetings)
+    .where(isNotNull(schema.meetings.recordingId));
+  conditions.push(notInArray(schema.recordings.id, meetingRecordingIds));
   if (folderId) {
     conditions.push(eq(schema.recordings.folderId, folderId));
   }
@@ -451,7 +489,7 @@ async function fetchShare(shareId: string) {
 
 export default defineAction({
   description:
-    "See what the user is currently looking at on screen. Returns the current navigation state plus relevant context (recording + transcript + comments on a recording page, folder contents on library, space list on spaces, etc.). Prefer reading the auto-included <current-screen> block — call this only when you need a refreshed snapshot.",
+    "See what the user is currently looking at on screen. Returns the current navigation state plus relevant context (recording + transcript + comments on a recording page, visible clips on library/shared-with-me, space list on spaces, etc.). Prefer reading the auto-included <current-screen> block — call this only when you need a refreshed snapshot.",
   schema: z.object({}),
   http: false,
   run: async () => {
@@ -531,7 +569,10 @@ export default defineAction({
       }
       case "library": {
         const [recordings, folders] = await Promise.all([
-          fetchLibrary(nav.folderId),
+          fetchLibrary({
+            folderId: nav.folderId,
+            organizationId,
+          }),
           fetchFoldersForSpace(null),
         ]);
         screen.library = {
@@ -543,6 +584,17 @@ export default defineAction({
         };
         break;
       }
+      case "shared": {
+        const recordings = await fetchLibrary({
+          shared: true,
+          organizationId,
+        });
+        screen.shared = {
+          count: recordings.length,
+          recordings,
+        };
+        break;
+      }
       case "spaces": {
         screen.spaces = await fetchSpaces(organizationId);
         break;

package/corpus/templates/clips/app/components/layout/Header.tsx

@@ -8,6 +8,7 @@ import { useLocation } from "react-router";
 const pageTitleKeys: Record<string, string> = {
   "/": "navigation.library",
   "/library": "navigation.library",
+  "/shared": "navigation.sharedWithMe",
   "/spaces": "navigation.spaces",
   "/archive": "navigation.archive",
   "/trash": "navigation.trash",

package/corpus/templates/clips/app/components/library/empty-state.tsx

@@ -13,6 +13,7 @@ import { Button } from "@/components/ui/button";
 
 type EmptyKind =
   | "library"
+  | "shared"
   | "folder"
   | "space"
   | "archive"
@@ -21,6 +22,7 @@ type EmptyKind =
 
 const ICONS: Record<EmptyKind, React.ComponentType<{ className?: string }>> = {
   library: IconVideo,
+  shared: IconUsersGroup,
   folder: IconFolder,
   space: IconUsersGroup,
   archive: IconArchive,