@agent-native/core 0.134.1 → 0.135.0

package/corpus/templates/clips/desktop/src/lib/recorder.ts

@@ -88,6 +88,13 @@ import {
   type CapturedTranscript,
   type TranscriptionCapture,
 } from "./transcription-capture";
+import {
+  buildStreamingReplayPlan,
+  planStreamingRecovery,
+  retryAttemptIdAfterRestartSignal,
+  retryAttemptIdAfterResumeResponse,
+  type UploadResumeResponse,
+} from "./upload-recovery";
 import {
   parseFinalizeReceipt,
   verifyFinalizeReceipt,
@@ -310,6 +317,7 @@ export interface PendingBrowserRecordingUpload {
   retryCount: number;
   chunkCount: number;
   mimeType: string;
+  uploadAttemptId?: string | null;
 }
 
 function streamFromTracks(tracks: MediaStreamTrack[]): MediaStream {
@@ -983,6 +991,16 @@ function buildRetryHeaders(mimeType: string, authToken?: string): Headers {
   return headers;
 }
 
+class UploadRestartRequiredError extends Error {
+  constructor(
+    message: string,
+    readonly recoveryEnabled?: boolean,
+  ) {
+    super(message);
+    this.name = "UploadRestartRequiredError";
+  }
+}
+
 async function postBackupChunk(
   url: string,
   blob: Blob,
@@ -999,6 +1017,24 @@ async function postBackupChunk(
   });
   const body = await res.text().catch(() => "");
   if (!res.ok) {
+    const details = (() => {
+      try {
+        return JSON.parse(body) as {
+          restartRequired?: unknown;
+          recoveryEnabled?: unknown;
+        };
+      } catch {
+        return null;
+      }
+    })();
+    if (details?.restartRequired === true) {
+      throw new UploadRestartRequiredError(
+        `The prior upload session expired (${res.status})`,
+        typeof details.recoveryEnabled === "boolean"
+          ? details.recoveryEnabled
+          : undefined,
+      );
+    }
     throw new Error(
       `Upload retry failed (${res.status}): ${body.slice(0, 200)}`,
     );
@@ -1009,7 +1045,9 @@ async function postBackupChunk(
 async function resetBrowserRecordingBackupUpload(
   meta: BrowserRecordingBackupMeta,
   authToken?: string,
-): Promise<UploadMode> {
+  attemptId?: string,
+  uploadGenerationId?: string,
+): Promise<{ uploadMode: UploadMode; uploadGenerationId?: string }> {
   const res = await fetch(
     `${meta.serverUrl.replace(/\/+$/, "")}/api/uploads/${meta.recordingId}/reset-chunks`,
     {
@@ -1023,6 +1061,8 @@ async function resetBrowserRecordingBackupUpload(
       body: JSON.stringify({
         requestStreaming: true,
         mimeType: meta.mimeType,
+        ...(attemptId ? { attemptId } : {}),
+        ...(uploadGenerationId ? { uploadGenerationId } : {}),
       }),
     },
   );
@@ -1034,14 +1074,61 @@ async function resetBrowserRecordingBackupUpload(
   }
   const body = (await res.json().catch(() => null)) as {
     uploadMode?: unknown;
+    uploadGenerationId?: unknown;
   } | null;
-  return body?.uploadMode === "streaming" ? "streaming" : "buffered";
+  if (attemptId && typeof body?.uploadGenerationId !== "string") {
+    throw new Error("Upload retry setup returned no upload generation");
+  }
+  return {
+    uploadMode: body?.uploadMode === "streaming" ? "streaming" : "buffered",
+    ...(typeof body?.uploadGenerationId === "string"
+      ? { uploadGenerationId: body.uploadGenerationId }
+      : {}),
+  };
+}
+
+async function getBrowserRecordingUploadResume(
+  meta: BrowserRecordingBackupMeta,
+  attemptId: string,
+  authToken?: string,
+): Promise<UploadResumeResponse> {
+  const resumeUrl = new URL(
+    `${meta.serverUrl.replace(/\/+$/, "")}/api/uploads/${meta.recordingId}/resume`,
+  );
+  resumeUrl.searchParams.set("attemptId", attemptId);
+  const res = await fetch(resumeUrl, {
+    method: "GET",
+    headers: buildRetryHeaders("application/json", authToken),
+    credentials: "include",
+  });
+  const body = await res.text().catch(() => "");
+  if (!res.ok) {
+    throw new Error(
+      `Upload resume check failed (${res.status}): ${body.slice(0, 200)}`,
+    );
+  }
+  let parsed: UploadResumeResponse;
+  try {
+    parsed = JSON.parse(body) as UploadResumeResponse;
+  } catch {
+    throw new Error("Upload resume check returned an unreadable response");
+  }
+  if (parsed.resumable && parsed.attemptId !== attemptId) {
+    throw new Error("Upload resume check returned a mismatched retry token");
+  }
+  return parsed;
 }
 
 async function replayBrowserBackupToResumableSession(
   meta: BrowserRecordingBackupMeta,
   chunks: BrowserRecordingBackupChunk[],
   authToken?: string,
+  attemptId?: string,
+  uploadGenerationId?: string,
+  resumeFrom: { bytesReceived: number; nextChunkIndex: number } = {
+    bytesReceived: 0,
+    nextChunkIndex: 0,
+  },
 ): Promise<FinalizeReceipt | null> {
   // The backup is stored in raw MediaRecorder blobs, which have arbitrary
   // boundaries. A resumable provider needs every non-final request aligned,
@@ -1056,33 +1143,38 @@ async function replayBrowserBackupToResumableSession(
     throw new Error("Local recording backup is empty");
   }
 
-  const fullChunks = Math.floor(recording.size / STREAM_CHUNK_BYTES);
-  const totalPosts = fullChunks + 1;
-  let offset = 0;
+  const replayPlan = buildStreamingReplayPlan({
+    localBytes: recording.size,
+    chunkBytes: STREAM_CHUNK_BYTES,
+    ...resumeFrom,
+  });
+  const totalPosts = Math.floor(recording.size / STREAM_CHUNK_BYTES) + 1;
 
-  for (let index = 0; index < fullChunks; index += 1) {
-    const body = recording.slice(
-      offset,
-      offset + STREAM_CHUNK_BYTES,
-      meta.mimeType,
-    );
+  for (const request of replayPlan.filter((item) => !item.final)) {
+    const body = recording.slice(request.start, request.end, meta.mimeType);
     await postBackupChunk(
-      chunkUrl(meta.serverUrl, meta.recordingId, index, false, {
+      chunkUrl(meta.serverUrl, meta.recordingId, request.index, false, {
         total: String(totalPosts),
         mimeType: meta.mimeType,
+        ...(attemptId ? { attemptId } : {}),
+        ...(uploadGenerationId ? { uploadGenerationId } : {}),
       }),
       body,
       authToken,
     );
-    offset += STREAM_CHUNK_BYTES;
   }
 
   // The final post always closes the session. When the file is exactly
   // aligned it is intentionally empty; the route sends the provider's close
   // request with the bytes committed by the previous chunks.
-  const finalBody = recording.slice(offset, recording.size, meta.mimeType);
+  const finalRequest = replayPlan[replayPlan.length - 1]!;
+  const finalBody = recording.slice(
+    finalRequest.start,
+    finalRequest.end,
+    meta.mimeType,
+  );
   return postBackupChunk(
-    chunkUrl(meta.serverUrl, meta.recordingId, fullChunks, true, {
+    chunkUrl(meta.serverUrl, meta.recordingId, finalRequest.index, true, {
       total: String(totalPosts),
       mimeType: meta.mimeType,
       durationMs: String(Math.round(meta.durationMs || 0)),
@@ -1090,6 +1182,8 @@ async function replayBrowserBackupToResumableSession(
       ...(meta.height ? { height: String(meta.height) } : {}),
       hasAudio: meta.hasAudio ? "1" : "0",
       hasCamera: meta.hasCamera ? "1" : "0",
+      ...(attemptId ? { attemptId } : {}),
+      ...(uploadGenerationId ? { uploadGenerationId } : {}),
     }),
     finalBody,
     authToken,
@@ -1100,6 +1194,10 @@ export async function retryBrowserRecordingBackup(input: {
   recordingId: string;
   serverUrl?: string;
   authToken?: string;
+  onRecoveryDecision?: (decision: {
+    action: "resume" | "restart" | "reconcile";
+    progress: number;
+  }) => void;
 }): Promise<{ recordingId: string; viewUrl: string }> {
   let meta = await getBrowserRecordingBackupMeta(input.recordingId);
   if (!meta) {
@@ -1111,57 +1209,165 @@ export async function retryBrowserRecordingBackup(input: {
   }
   const chunks = await getBrowserRecordingBackupChunks(input.recordingId);
   const validatedChunks = validateBrowserRecordingBackupChunks(meta, chunks);
+  let activeAttemptId: string | undefined =
+    meta.uploadAttemptId || crypto.randomUUID();
+  let activeUploadGenerationId: string | undefined;
 
   try {
     await putBrowserRecordingBackupMeta({
       ...meta,
+      uploadAttemptId: activeAttemptId,
       lastAttemptAt: new Date().toISOString(),
       lastError: null,
     });
-    const uploadMode = await resetBrowserRecordingBackupUpload(
+    const recordingBytes = validatedChunks.reduce(
+      (total, chunk) => total + chunk.blob.size,
+      0,
+    );
+    const resumeResponse = await getBrowserRecordingUploadResume(
       meta,
+      activeAttemptId,
       input.authToken,
     );
+    const recoveryPlan = planStreamingRecovery({
+      response: resumeResponse,
+      localBytes: recordingBytes,
+      chunkBytes: STREAM_CHUNK_BYTES,
+    });
+    activeAttemptId = retryAttemptIdAfterResumeResponse(
+      activeAttemptId,
+      resumeResponse,
+    );
+    activeUploadGenerationId = resumeResponse.resumable
+      ? resumeResponse.uploadGenerationId
+      : undefined;
+    if (recoveryPlan.action === "reconcile") {
+      input.onRecoveryDecision?.({ action: "reconcile", progress: 1 });
+      if (
+        await recoverAcceptedRecordingAfterFinalizeError({
+          serverUrl: meta.serverUrl,
+          recordingId: meta.recordingId,
+          authToken: input.authToken,
+        })
+      ) {
+        return {
+          recordingId: meta.recordingId,
+          viewUrl: `/r/${meta.recordingId}`,
+        };
+      }
+      throw new Error(
+        `Upload is ${recoveryPlan.status}, but its accepted media could not be verified`,
+      );
+    }
+
+    let uploadMode: UploadMode = "streaming";
+    let resumeFrom = { bytesReceived: 0, nextChunkIndex: 0 };
+    if (recoveryPlan.action === "resume") {
+      resumeFrom = recoveryPlan;
+      input.onRecoveryDecision?.({
+        action: "resume",
+        progress: recoveryPlan.progress,
+      });
+      console.info("[clips-recorder] resuming saved upload", {
+        recordingId: meta.recordingId,
+        localBytes: recordingBytes,
+        serverAcknowledgedBytes: recoveryPlan.bytesReceived,
+        nextChunkIndex: recoveryPlan.nextChunkIndex,
+        progress: recoveryPlan.progress,
+      });
+    } else {
+      input.onRecoveryDecision?.({ action: "restart", progress: 0 });
+      console.info("[clips-recorder] restarting saved upload", {
+        recordingId: meta.recordingId,
+        localBytes: recordingBytes,
+        reason: recoveryPlan.reason,
+      });
+      const reset = await resetBrowserRecordingBackupUpload(
+        meta,
+        input.authToken,
+        activeAttemptId,
+        activeUploadGenerationId,
+      );
+      uploadMode = reset.uploadMode;
+      activeUploadGenerationId = reset.uploadGenerationId;
+    }
 
     if (uploadMode === "streaming") {
+      let receipt: FinalizeReceipt | null = null;
       try {
-        const receipt = await replayBrowserBackupToResumableSession(
+        receipt = await replayBrowserBackupToResumableSession(
           meta,
           validatedChunks,
           input.authToken,
+          activeAttemptId,
+          activeUploadGenerationId,
+          resumeFrom,
         );
-        const receiptStatus = verifyFinalizeReceipt(receipt, meta);
-        if (receiptStatus === "processing") {
-          scheduleBrowserBackupCleanupAfterProcessing({
+      } catch (err) {
+        if (err instanceof UploadRestartRequiredError) {
+          activeAttemptId = retryAttemptIdAfterRestartSignal(
+            activeAttemptId,
+            err.recoveryEnabled,
+          );
+          if (err.recoveryEnabled === false) {
+            activeUploadGenerationId = undefined;
+          }
+          input.onRecoveryDecision?.({ action: "restart", progress: 0 });
+          console.info("[clips-recorder] restarting expired upload session", {
+            recordingId: meta.recordingId,
+            localBytes: recordingBytes,
+          });
+          const reset = await resetBrowserRecordingBackupUpload(
+            meta,
+            input.authToken,
+            activeAttemptId,
+            activeUploadGenerationId,
+          );
+          uploadMode = reset.uploadMode;
+          activeUploadGenerationId = reset.uploadGenerationId;
+          if (uploadMode === "streaming") {
+            receipt = await replayBrowserBackupToResumableSession(
+              meta,
+              validatedChunks,
+              input.authToken,
+              activeAttemptId,
+              activeUploadGenerationId,
+            );
+          }
+        } else if (
+          await recoverAcceptedRecordingAfterFinalizeError({
             serverUrl: meta.serverUrl,
             recordingId: meta.recordingId,
             authToken: input.authToken,
-          });
+          })
+        ) {
           return {
             recordingId: meta.recordingId,
             viewUrl: `/r/${meta.recordingId}`,
           };
+        } else {
+          throw err;
         }
-      } catch (err) {
-        if (
-          await recoverAcceptedRecordingAfterFinalizeError({
+      }
+      if (uploadMode === "streaming") {
+        const receiptStatus = verifyFinalizeReceipt(receipt, meta);
+        if (receiptStatus === "processing") {
+          scheduleBrowserBackupCleanupAfterProcessing({
             serverUrl: meta.serverUrl,
             recordingId: meta.recordingId,
             authToken: input.authToken,
-          })
-        ) {
+          });
           return {
             recordingId: meta.recordingId,
             viewUrl: `/r/${meta.recordingId}`,
           };
         }
-        throw err;
+        await deleteBrowserRecordingBackup(meta.recordingId);
+        return {
+          recordingId: meta.recordingId,
+          viewUrl: `/r/${meta.recordingId}`,
+        };
       }
-      await deleteBrowserRecordingBackup(meta.recordingId);
-      return {
-        recordingId: meta.recordingId,
-        viewUrl: `/r/${meta.recordingId}`,
-      };
     }
 
     const totalPosts = validatedChunks.length + 1;
@@ -1170,6 +1376,10 @@ export async function retryBrowserRecordingBackup(input: {
         chunkUrl(meta.serverUrl, meta.recordingId, chunk.index, false, {
           total: String(totalPosts),
           mimeType: meta.mimeType,
+          ...(activeAttemptId ? { attemptId: activeAttemptId } : {}),
+          ...(activeUploadGenerationId
+            ? { uploadGenerationId: activeUploadGenerationId }
+            : {}),
         }),
         chunk.blob,
         input.authToken,
@@ -1189,6 +1399,10 @@ export async function retryBrowserRecordingBackup(input: {
         ...(meta.height ? { height: String(meta.height) } : {}),
         hasAudio: meta.hasAudio ? "1" : "0",
         hasCamera: meta.hasCamera ? "1" : "0",
+        ...(activeAttemptId ? { attemptId: activeAttemptId } : {}),
+        ...(activeUploadGenerationId
+          ? { uploadGenerationId: activeUploadGenerationId }
+          : {}),
       },
     );
     try {
@@ -1229,9 +1443,29 @@ export async function retryBrowserRecordingBackup(input: {
     return { recordingId: meta.recordingId, viewUrl: `/r/${meta.recordingId}` };
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
+    if (
+      await recoverAcceptedRecordingAfterFinalizeError({
+        serverUrl: meta.serverUrl,
+        recordingId: meta.recordingId,
+        authToken: input.authToken,
+      })
+    ) {
+      return {
+        recordingId: meta.recordingId,
+        viewUrl: `/r/${meta.recordingId}`,
+      };
+    }
     await markBrowserRecordingBackupError(meta.recordingId, message).catch(
       () => {},
     );
+    await interruptRecordingUpload(
+      meta.serverUrl,
+      meta.recordingId,
+      message,
+      input.authToken,
+      activeAttemptId,
+      activeUploadGenerationId,
+    );
     throw err;
   }
 }
@@ -1610,6 +1844,55 @@ async function abortRecordingUpload(
   }
 }
 
+async function interruptRecordingUpload(
+  serverUrl: string,
+  recordingId: string,
+  detail: string,
+  authToken?: string,
+  attemptId?: string,
+  uploadGenerationId?: string,
+): Promise<void> {
+  let lastError: unknown = null;
+  for (let attempt = 1; attempt <= CHUNK_UPLOAD_MAX_ATTEMPTS; attempt += 1) {
+    try {
+      const res = await fetch(
+        `${serverUrl.replace(/\/+$/, "")}/api/uploads/${recordingId}/interrupt`,
+        {
+          method: "POST",
+          headers: buildRetryHeaders("application/json", authToken),
+          credentials: "include",
+          body: JSON.stringify({
+            detail,
+            ...(attemptId ? { attemptId } : {}),
+            ...(uploadGenerationId ? { uploadGenerationId } : {}),
+          }),
+        },
+      );
+      if (res.ok) {
+        console.info("[clips-recorder] upload interruption recorded", {
+          recordingId,
+          attempt,
+        });
+        return;
+      }
+      const body = await res.text().catch(() => "");
+      lastError = new Error(
+        `Upload interruption failed (${res.status}): ${body.slice(0, 200)}`,
+      );
+      if (!isRetriableChunkStatus(res.status)) break;
+    } catch (err) {
+      lastError = err;
+    }
+    if (attempt < CHUNK_UPLOAD_MAX_ATTEMPTS) {
+      await wait(CHUNK_UPLOAD_RETRY_BASE_MS * 2 ** (attempt - 1));
+    }
+  }
+  console.warn(
+    "[clips-recorder] upload interruption could not be recorded:",
+    lastError,
+  );
+}
+
 async function trashRecording(
   serverUrl: string,
   recordingId: string,
@@ -2427,10 +2710,11 @@ async function tryStartRewindFullscreenRecording(
             ) {
               return { recordingId: id, viewUrl };
             }
-            await abortRecordingUpload(
+            await interruptRecordingUpload(
               params.serverUrl,
               id,
               err instanceof Error ? err.message : String(err),
+              params.authToken,
             );
             throw err;
           }
@@ -3101,10 +3385,11 @@ async function startNativeFullscreenRecording(
             ) {
               return { recordingId: id, viewUrl };
             }
-            await abortRecordingUpload(
+            await interruptRecordingUpload(
               params.serverUrl,
               id,
               err instanceof Error ? err.message : String(err),
+              params.authToken,
             );
             throw err;
           }
@@ -4453,7 +4738,12 @@ async function startRecordingInner(
           await markBrowserRecordingBackupError(id, failed.message).catch(
             () => {},
           );
-          await abortRecordingUpload(params.serverUrl, id, failed.message);
+          await interruptRecordingUpload(
+            params.serverUrl,
+            id,
+            failed.message,
+            params.authToken,
+          );
         } finally {
           await clearRecordingState();
           await publishFinalizingResult({
@@ -4559,7 +4849,12 @@ async function startRecordingInner(
               await markBrowserRecordingBackupError(id, error.message).catch(
                 () => {},
               );
-              await abortRecordingUpload(params.serverUrl, id, error.message);
+              await interruptRecordingUpload(
+                params.serverUrl,
+                id,
+                error.message,
+                params.authToken,
+              );
               throw error;
             }
             await deleteBrowserRecordingBackup(id).catch((err) => {

package/corpus/templates/clips/desktop/src/lib/upload-recovery.ts

@@ -0,0 +1,129 @@
+export type UploadResumeResponse =
+  | {
+      resumable: true;
+      recoveryEnabled: boolean;
+      status: string;
+      uploadMode: "streaming" | "buffered";
+      attemptId: string;
+      uploadGenerationId?: string;
+      bytesReceived: number;
+      nextChunkIndex: number;
+    }
+  | {
+      resumable: false;
+      recoveryEnabled: boolean;
+      status: string | null;
+      failureReason?: string | null;
+      videoUrl?: string | null;
+      reason?: string;
+      attemptId?: string;
+    };
+
+export type StreamingRecoveryPlan =
+  | {
+      action: "resume";
+      bytesReceived: number;
+      nextChunkIndex: number;
+      progress: number;
+    }
+  | { action: "reconcile"; status: "ready" | "processing" }
+  | { action: "restart"; reason: string };
+
+export interface StreamingReplayRequest {
+  index: number;
+  start: number;
+  end: number;
+  final: boolean;
+}
+
+export function retryAttemptIdAfterRestartSignal(
+  attemptId: string | undefined,
+  recoveryEnabled: unknown,
+): string | undefined {
+  return recoveryEnabled === false ? undefined : attemptId;
+}
+
+export function retryAttemptIdAfterResumeResponse(
+  attemptId: string | undefined,
+  response: UploadResumeResponse,
+): string | undefined {
+  return response.resumable && response.attemptId === attemptId
+    ? attemptId
+    : undefined;
+}
+
+export function buildStreamingReplayPlan(input: {
+  localBytes: number;
+  chunkBytes: number;
+  bytesReceived: number;
+  nextChunkIndex: number;
+}): StreamingReplayRequest[] {
+  const { localBytes, chunkBytes, bytesReceived, nextChunkIndex } = input;
+  const fullChunks = Math.floor(localBytes / chunkBytes);
+  const requests: StreamingReplayRequest[] = [];
+  let offset = bytesReceived;
+  for (let index = nextChunkIndex; index < fullChunks; index += 1) {
+    requests.push({
+      index,
+      start: offset,
+      end: offset + chunkBytes,
+      final: false,
+    });
+    offset += chunkBytes;
+  }
+  requests.push({
+    index: fullChunks,
+    start: offset,
+    end: localBytes,
+    final: true,
+  });
+  return requests;
+}
+
+export function planStreamingRecovery(input: {
+  response: UploadResumeResponse;
+  localBytes: number;
+  chunkBytes: number;
+}): StreamingRecoveryPlan {
+  const { response, localBytes, chunkBytes } = input;
+  if (!response.recoveryEnabled) {
+    return { action: "restart", reason: "resumable retry is disabled" };
+  }
+  if (response.status === "ready" || response.status === "processing") {
+    return { action: "reconcile", status: response.status };
+  }
+  if (!response.resumable) {
+    return {
+      action: "restart",
+      reason: response.reason ?? "server session is unavailable",
+    };
+  }
+  if (response.uploadMode !== "streaming") {
+    return { action: "restart", reason: "server session is not streaming" };
+  }
+  if (!response.attemptId) {
+    return { action: "restart", reason: "server retry token is missing" };
+  }
+  const { bytesReceived, nextChunkIndex } = response;
+  if (
+    !Number.isSafeInteger(bytesReceived) ||
+    bytesReceived < 0 ||
+    bytesReceived > localBytes
+  ) {
+    return { action: "restart", reason: "server byte offset is invalid" };
+  }
+  if (
+    !Number.isSafeInteger(nextChunkIndex) ||
+    nextChunkIndex < 0 ||
+    bytesReceived % chunkBytes !== 0 ||
+    nextChunkIndex !== bytesReceived / chunkBytes
+  ) {
+    return { action: "restart", reason: "server chunk offset is inconsistent" };
+  }
+  return {
+    action: "resume",
+    bytesReceived,
+    nextChunkIndex,
+    progress: localBytes > 0 ? bytesReceived / localBytes : 0,
+  };
+}