@agent-native/core 0.134.0 → 0.135.0

package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/interrupt.post.ts

@@ -0,0 +1,197 @@
+// guard:allow-api-route — Upload transport interruption preserves resumable state for desktop recovery.
+
+/**
+ * Mark a live upload as interrupted without discarding its provider session or
+ * buffered chunks. Explicit cancellation continues to use /abort.
+ *
+ * Route: POST /api/uploads/:recordingId/interrupt
+ */
+
+import {
+  compareAndSetAppState,
+  readAppState,
+  writeAppState,
+} from "@agent-native/core/application-state";
+import { isFeatureFlagEnabled } from "@agent-native/core/feature-flags";
+import { runWithRequestContext } from "@agent-native/core/server";
+import { and, eq, isNull } from "drizzle-orm";
+import {
+  defineEventHandler,
+  getRouterParam,
+  readBody,
+  setResponseStatus,
+  type H3Event,
+} from "h3";
+
+import { UPLOAD_RETRY_RESUME_FLAG } from "../../../../../shared/feature-flags.js";
+import { getDb, schema } from "../../../../db/index.js";
+import {
+  getEventOwnerContext,
+  ownerEmailMatches,
+} from "../../../../lib/recordings.js";
+import {
+  isRetryableUploadInterruption,
+  RETRYABLE_UPLOAD_INTERRUPTION_REASON,
+} from "../../../../lib/upload-interruption.js";
+import abortUpload from "./abort.post.js";
+
+export default defineEventHandler(async (event: H3Event) => {
+  const recordingId = getRouterParam(event, "recordingId");
+  if (!recordingId) {
+    setResponseStatus(event, 400);
+    return { error: "Missing recordingId" };
+  }
+
+  const { userEmail: ownerEmail, orgId } = await getEventOwnerContext(event);
+  if (
+    !(await isFeatureFlagEnabled(UPLOAD_RETRY_RESUME_FLAG, {
+      userEmail: ownerEmail,
+      userKey: ownerEmail,
+      orgId,
+    }))
+  ) {
+    return abortUpload(event);
+  }
+  const body = (await readBody(event).catch(() => null)) as {
+    detail?: unknown;
+    attemptId?: unknown;
+    uploadGenerationId?: unknown;
+  } | null;
+  const attemptId =
+    typeof body?.attemptId === "string" &&
+    body.attemptId.length > 0 &&
+    body.attemptId.length <= 128
+      ? body.attemptId
+      : null;
+  const interruptionDetail =
+    typeof body?.detail === "string" && body.detail.trim()
+      ? body.detail.trim().slice(0, 1000)
+      : null;
+  const uploadGenerationId =
+    typeof body?.uploadGenerationId === "string" &&
+    body.uploadGenerationId.length > 0 &&
+    body.uploadGenerationId.length <= 128
+      ? body.uploadGenerationId
+      : null;
+
+  return runWithRequestContext({ userEmail: ownerEmail, orgId }, async () => {
+    const db = getDb();
+    const [existing] = await db
+      .select({
+        id: schema.recordings.id,
+        status: schema.recordings.status,
+        failureReason: schema.recordings.failureReason,
+        videoUrl: schema.recordings.videoUrl,
+        uploadAttemptId: schema.recordings.uploadAttemptId,
+        uploadGenerationId: schema.recordings.uploadGenerationId,
+      })
+      .from(schema.recordings)
+      .where(
+        and(
+          eq(schema.recordings.id, recordingId),
+          ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+        ),
+      );
+
+    if (!existing) {
+      setResponseStatus(event, 404);
+      return { error: "Recording not found" };
+    }
+    if (existing.status === "ready" || existing.status === "processing") {
+      return {
+        ok: true,
+        recordingId,
+        status: existing.status,
+        alreadyAccepted: true,
+      };
+    }
+    if ((existing.uploadAttemptId ?? null) !== attemptId) {
+      setResponseStatus(event, 409);
+      return {
+        error: "A newer upload retry is already active.",
+        staleAttempt: true,
+      };
+    }
+    if ((existing.uploadGenerationId ?? null) !== uploadGenerationId) {
+      setResponseStatus(event, 409);
+      return {
+        error: "A newer upload generation is already active.",
+        staleAttempt: true,
+      };
+    }
+    const alreadyInterrupted =
+      existing.status === "failed" &&
+      isRetryableUploadInterruption(existing.failureReason);
+    if (existing.status !== "uploading" && !alreadyInterrupted) {
+      setResponseStatus(event, 409);
+      return { error: "Recording upload is no longer active" };
+    }
+
+    const uploadStateKey = `recording-upload-${recordingId}`;
+    const uploadStateRaw = await readAppState(uploadStateKey);
+    const uploadState = uploadStateRaw ?? {};
+    const interruptedAt = new Date().toISOString();
+    if (!alreadyInterrupted) {
+      const interrupted = await db
+        .update(schema.recordings)
+        .set({
+          status: "failed",
+          failureReason: RETRYABLE_UPLOAD_INTERRUPTION_REASON,
+          updatedAt: interruptedAt,
+        })
+        .where(
+          and(
+            eq(schema.recordings.id, recordingId),
+            ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+            eq(schema.recordings.status, "uploading"),
+            attemptId === null
+              ? isNull(schema.recordings.uploadAttemptId)
+              : eq(schema.recordings.uploadAttemptId, attemptId),
+            uploadGenerationId === null
+              ? isNull(schema.recordings.uploadGenerationId)
+              : eq(schema.recordings.uploadGenerationId, uploadGenerationId),
+          ),
+        )
+        .returning({ id: schema.recordings.id });
+
+      if (interrupted.length !== 1) {
+        setResponseStatus(event, 409);
+        return { error: "Recording upload changed while it was interrupted" };
+      }
+    }
+
+    const uploadStateUpdated = await compareAndSetAppState(
+      uploadStateKey,
+      uploadStateRaw,
+      {
+        ...uploadState,
+        recordingId,
+        status: "failed",
+        failureReason: RETRYABLE_UPLOAD_INTERRUPTION_REASON,
+        retryableInterruption: true,
+        uploadAttemptId: attemptId,
+        uploadGenerationId,
+        ...(interruptionDetail ? { interruptionDetail } : {}),
+        updatedAt: interruptedAt,
+      },
+    );
+    if (!uploadStateUpdated) {
+      console.info(
+        `[upload-interrupt] upload state changed after interruption claim; preserving replacement state for ${recordingId}`,
+      );
+    }
+    await writeAppState("refresh-signal", { ts: Date.now() });
+
+    console.info("[upload-interrupt] resumable state preserved", {
+      recordingId,
+      hasInterruptionDetail: interruptionDetail !== null,
+    });
+    return {
+      ok: true,
+      recordingId,
+      status: "failed",
+      resumable: true,
+      ...(alreadyInterrupted ? { alreadyInterrupted: true } : {}),
+    };
+  });
+});

package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/reset-chunks.post.ts

@@ -24,15 +24,19 @@
  * Route: POST /api/uploads/:recordingId/reset-chunks
  */
 
+import { randomUUID } from "node:crypto";
+
 import {
+  compareAndSetAppState,
+  readAppState,
   writeAppState,
-  deleteAppStateByPrefix,
 } from "@agent-native/core/application-state";
+import { isFeatureFlagEnabled } from "@agent-native/core/feature-flags";
 import { getActiveFileUploadProviderForRequest } from "@agent-native/core/file-upload";
 import { runWithRequestContext } from "@agent-native/core/server";
 import type { UploadMode } from "@shared/recording-core.js";
 import { MAX_UPLOAD_BYTES as MAX_RECORDING_UPLOAD_BYTES } from "@shared/upload-limits.js";
-import { and, eq } from "drizzle-orm";
+import { and, eq, isNull } from "drizzle-orm";
 import {
   defineEventHandler,
   getRouterParam,
@@ -41,8 +45,10 @@ import {
   type H3Event,
 } from "h3";
 
+import { UPLOAD_RETRY_RESUME_FLAG } from "../../../../../shared/feature-flags.js";
 import { getDb, schema } from "../../../../db/index.js";
 import { isMediaVerificationPending } from "../../../../lib/media-verification-state.js";
+import { deleteRecordingChunks } from "../../../../lib/recording-upload-state.js";
 import {
   getEventOwnerContext,
   ownerEmailMatches,
@@ -52,7 +58,10 @@ import {
   setResumableSession,
 } from "../../../../lib/resumable-session.js";
 import { shouldEnableStreamingUpload } from "../../../../lib/streaming-upload-mode.js";
-import { uploadLeaseExpiry } from "../../../../lib/upload-lease.js";
+import {
+  renewUploadLease,
+  uploadLeaseExpiry,
+} from "../../../../lib/upload-lease.js";
 import { allowsSqlRecordingChunkScratch } from "../../../../lib/video-storage.js";
 
 interface CompressionMeta {
@@ -98,7 +107,15 @@ export default defineEventHandler(async (event: H3Event) => {
     compression?: CompressionMeta | null;
     requestStreaming?: boolean;
     mimeType?: string;
+    attemptId?: string;
+    uploadGenerationId?: string;
+    useGenerationFence?: boolean;
   } | null;
+  const recoveryEnabled = await isFeatureFlagEnabled(UPLOAD_RETRY_RESUME_FLAG, {
+    userEmail: ownerEmail,
+    userKey: ownerEmail,
+    orgId,
+  });
 
   // Sanitize compression metadata. The recorder is the only client we trust
   // here, but the values land in Sentry extras — so we still bound them to
@@ -121,6 +138,8 @@ export default defineEventHandler(async (event: H3Event) => {
         id: schema.recordings.id,
         status: schema.recordings.status,
         videoUrl: schema.recordings.videoUrl,
+        uploadAttemptId: schema.recordings.uploadAttemptId,
+        uploadGenerationId: schema.recordings.uploadGenerationId,
       })
       .from(schema.recordings)
       .where(
@@ -140,6 +159,35 @@ export default defineEventHandler(async (event: H3Event) => {
       return { error: "Recording is already ready" };
     }
 
+    const requestedAttemptId =
+      typeof body?.attemptId === "string" &&
+      body.attemptId.length > 0 &&
+      body.attemptId.length <= 128
+        ? body.attemptId
+        : null;
+    const existingAttemptId = existing.uploadAttemptId ?? null;
+    const existingGenerationId = existing.uploadGenerationId ?? null;
+    if (recoveryEnabled && existingAttemptId !== requestedAttemptId) {
+      setResponseStatus(event, 409);
+      return {
+        error: "A newer upload retry is already active.",
+        staleAttempt: true,
+      };
+    }
+    const requestedGenerationId =
+      typeof body?.uploadGenerationId === "string" &&
+      body.uploadGenerationId.length > 0 &&
+      body.uploadGenerationId.length <= 128
+        ? body.uploadGenerationId
+        : null;
+    if (recoveryEnabled && existingGenerationId !== requestedGenerationId) {
+      setResponseStatus(event, 409);
+      return {
+        error: "A newer upload generation is already active.",
+        staleAttempt: true,
+      };
+    }
+
     if (
       await isMediaVerificationPending({
         ownerEmail,
@@ -150,15 +198,72 @@ export default defineEventHandler(async (event: H3Event) => {
       setResponseStatus(event, 409);
       return { error: "Recording is still being verified" };
     }
+    if (existing.status !== "uploading" && existing.status !== "failed") {
+      setResponseStatus(event, 409);
+      return { error: "Recording upload is no longer resettable" };
+    }
 
-    const cleared = await deleteAppStateByPrefix(
-      `recording-chunks-${recordingId}-`,
+    // Fence this reset before deleting any provider or buffered state. A
+    // retry that lost the token race must not tear down the winner's session.
+    const now = new Date().toISOString();
+    // Only clients that can carry the returned generation may opt into the
+    // fence. Retry claims always opt in; legacy reset callers keep the null
+    // generation wire contract until they are upgraded.
+    const useGenerationFence =
+      recoveryEnabled &&
+      (requestedAttemptId !== null ||
+        requestedGenerationId !== null ||
+        body?.useGenerationFence === true);
+    const nextGenerationId = useGenerationFence ? randomUUID() : null;
+    const uploadStateKey = `recording-upload-${recordingId}`;
+    const uploadStateSnapshot = await readAppState(uploadStateKey);
+    const reset = await db
+      .update(schema.recordings)
+      .set({
+        status: "uploading",
+        failureReason: null,
+        uploadProgress: 0,
+        uploadGenerationId: nextGenerationId,
+        ...(!recoveryEnabled ? { uploadAttemptId: null } : {}),
+        uploadLeaseExpiresAt: uploadLeaseExpiry(),
+        updatedAt: now,
+      })
+      .where(
+        and(
+          eq(schema.recordings.id, recordingId),
+          ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+          eq(schema.recordings.status, existing.status),
+          existingAttemptId === null
+            ? isNull(schema.recordings.uploadAttemptId)
+            : eq(schema.recordings.uploadAttemptId, existingAttemptId),
+          existingGenerationId === null
+            ? isNull(schema.recordings.uploadGenerationId)
+            : eq(schema.recordings.uploadGenerationId, existingGenerationId),
+        ),
+      )
+      .returning({ id: schema.recordings.id });
+
+    if (reset.length !== 1) {
+      setResponseStatus(event, 409);
+      return {
+        error: "A newer upload retry is already active.",
+        staleAttempt: true,
+      };
+    }
+
+    const cleared = await deleteRecordingChunks(
+      ownerEmail,
+      recordingId,
+      existingGenerationId,
     );
     // Clear any stale resumable session so a buffered retry does not
     // accidentally route through handleResumableChunk with stale offsets.
-    await deleteResumableSession(recordingId).catch(() => {});
+    await deleteResumableSession(recordingId, existingGenerationId).catch(
+      () => {},
+    );
 
     let uploadMode: UploadMode = "buffered";
+    let compensateStartedSession: (() => Promise<void>) | null = null;
     if (body?.requestStreaming === true) {
       const mimeType = normalizeVideoMimeType(body.mimeType);
       if (!mimeType) {
@@ -184,17 +289,33 @@ export default defineEventHandler(async (event: H3Event) => {
             mimeType,
             MAX_RECORDING_UPLOAD_BYTES,
           );
-          await setResumableSession(recordingId, {
-            providerId: uploadProvider.id,
-            sessionId: session.sessionId,
-            meta: {
-              ...session.meta,
-              stableUrl: true,
-              recordAsset: false,
+          await setResumableSession(
+            recordingId,
+            {
+              providerId: uploadProvider.id,
+              sessionId: session.sessionId,
+              meta: {
+                ...session.meta,
+                stableUrl: true,
+                recordAsset: false,
+              },
+              bytesUploaded: 0,
+              lastCommittedIndex: -1,
             },
-            bytesUploaded: 0,
-            lastCommittedIndex: -1,
-          });
+            nextGenerationId,
+          );
+          compensateStartedSession = async () => {
+            if (!uploadProvider.resumable?.abortSession) {
+              throw new Error(
+                `Resumable upload provider ${uploadProvider.id} cannot abort the discarded retry session`,
+              );
+            }
+            await uploadProvider.resumable.abortSession({
+              sessionId: session.sessionId,
+              meta: session.meta,
+            });
+            await deleteResumableSession(recordingId, nextGenerationId);
+          };
           uploadMode = "streaming";
         } catch (err) {
           if (!bufferedFallbackAvailable) {
@@ -219,19 +340,64 @@ export default defineEventHandler(async (event: H3Event) => {
       }
     }
 
+    const resetLease = await renewUploadLease(recordingId, {
+      attemptId: recoveryEnabled ? existingAttemptId : null,
+      generationId: nextGenerationId,
+    });
+    if (!resetLease.held) {
+      await compensateStartedSession?.();
+      setResponseStatus(event, 409);
+      return {
+        error: "Recording upload changed while its retry was starting.",
+        staleAttempt: true,
+      };
+    }
+
     // Reset the per-recording upload progress so the UI poller sees the
     // re-upload restart from 0 and doesn't briefly show "100% then
     // re-running" on the post-compression chunked upload pass.
-    const now = new Date().toISOString();
-    await writeAppState(`recording-upload-${recordingId}`, {
-      recordingId,
-      status: "uploading",
-      progress: 0,
-      chunksReceived: 0,
-      bytesReceived: 0,
-      maxBytes: MAX_RECORDING_UPLOAD_BYTES,
-      updatedAt: now,
-    });
+    const uploadStateUpdated = await compareAndSetAppState(
+      uploadStateKey,
+      uploadStateSnapshot,
+      {
+        recordingId,
+        status: "uploading",
+        progress: 0,
+        chunksReceived: 0,
+        bytesReceived: 0,
+        uploadAttemptId: recoveryEnabled ? existingAttemptId : null,
+        uploadGenerationId: nextGenerationId,
+        maxBytes: MAX_RECORDING_UPLOAD_BYTES,
+        updatedAt: now,
+      },
+    );
+    if (!uploadStateUpdated) {
+      const [current] = await db
+        .select({
+          status: schema.recordings.status,
+          uploadAttemptId: schema.recordings.uploadAttemptId,
+          uploadGenerationId: schema.recordings.uploadGenerationId,
+        })
+        .from(schema.recordings)
+        .where(
+          and(
+            eq(schema.recordings.id, recordingId),
+            ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
+          ),
+        );
+      if (
+        current?.status !== "uploading" ||
+        (current.uploadAttemptId ?? null) !==
+          (recoveryEnabled ? existingAttemptId : null) ||
+        (current.uploadGenerationId ?? null) !== nextGenerationId
+      ) {
+        setResponseStatus(event, 409);
+        return {
+          error: "Recording upload changed while its retry was starting.",
+          staleAttempt: true,
+        };
+      }
+    }
 
     // Stash compression metadata under its own key. We don't merge it into
     // `recording-upload-{id}` because the recorder client overwrites that
@@ -245,25 +411,13 @@ export default defineEventHandler(async (event: H3Event) => {
       });
     }
 
-    await db
-      .update(schema.recordings)
-      .set({
-        status: "uploading",
-        failureReason: null,
-        uploadProgress: 0,
-        // A retry re-takes the lease. Without this the row would go back to
-        // 'uploading' still carrying the expired lease that killed it.
-        uploadLeaseExpiresAt: uploadLeaseExpiry(),
-        updatedAt: now,
-      })
-      .where(eq(schema.recordings.id, recordingId));
-
     return {
       ok: true,
       recordingId,
       chunksCleared: cleared,
       compressionRecorded: !!compression,
       uploadMode,
+      uploadGenerationId: nextGenerationId,
     };
   });
 });