@agent-native/core 0.85.6 → 0.85.7

package/corpus/templates/clips/desktop/src/lib/recorder.ts

@@ -88,6 +88,17 @@ const DEV_SYNTHETIC_CAPTURE_FLAG = "clips:dev-synthetic-capture";
 const LEGACY_DEV_REAL_CAPTURE_FLAG = "clips:dev-real-capture";
 const LIVE_UPLOAD_CHUNK_MS = 1_000;
 const NATIVE_FULLSCREEN_SEGMENT_MS = 5 * 60_000;
+// GCS resumable uploads require every non-final chunk to be a multiple of
+// 256 KiB. MediaRecorder emits arbitrary blob sizes, so on the streaming path
+// we buffer raw blobs and only PUT aligned slices; the unaligned remainder is
+// held and sent as the final chunk on stop.
+const GCS_CHUNK_ALIGN_BYTES = 256 * 1024;
+const STREAM_CHUNK_BYTES = 15 * GCS_CHUNK_ALIGN_BYTES; // 3.75 MiB
+
+// How the client delivers recorded data to the server.
+//  - "streaming" — server has a resumable session; flush aligned chunks live.
+//  - "buffered"  — per-blob chunks staged server-side, assembled on finalize.
+type UploadMode = "streaming" | "buffered";
 const CLOUD_CAPTURE_FRAME_RATE = 24;
 const CLOUD_CAPTURE_MAX_WIDTH = 1920;
 const CLOUD_CAPTURE_MAX_HEIGHT = 1080;
@@ -903,12 +914,14 @@ async function createServerRecording(
   hasCamera: boolean,
   hasAudio: boolean,
   titleContext?: CaptureTitleResult,
+  options?: { mimeType?: string; requestStreaming?: boolean },
 ) {
   const url = `${serverUrl.replace(/\/+$/, "")}/_agent-native/actions/create-recording`;
   console.log("[clips-recorder] POST", url, {
     hasCamera,
     hasAudio,
     title: titleContext?.title,
+    requestStreaming: options?.requestStreaming ?? false,
   });
   let res: Response;
   try {
@@ -925,6 +938,9 @@ async function createServerRecording(
         hasAudio,
         spaceIds: [],
         visibility: "public",
+        ...(options?.requestStreaming
+          ? { requestStreaming: true, mimeType: options.mimeType }
+          : {}),
         ...(titleContext
           ? {
               title: titleContext.title,
@@ -947,12 +963,18 @@ async function createServerRecording(
     console.error("[clips-recorder] bad response:", url, res.status, body);
     throw new Error(`create-recording ${res.status}: ${body.slice(0, 200)}`);
   }
-  const data = (await res.json()) as { result?: { id: string }; id?: string };
+  const data = (await res.json()) as {
+    result?: { id: string; uploadMode?: string };
+    id?: string;
+    uploadMode?: string;
+  };
   const result = data.result ?? data;
   if (!result.id) {
     throw new Error("create-recording did not return an id");
   }
-  return { id: result.id };
+  const uploadMode: UploadMode =
+    result.uploadMode === "streaming" ? "streaming" : "buffered";
+  return { id: result.id, uploadMode };
 }
 
 interface ActiveWindowContext {
@@ -1234,48 +1256,95 @@ function decChunkBusy(): void {
   }
 }
 
+// Bounded retry for live chunk uploads. A brief network blip (Wi-Fi roam, DNS
+// hiccup, a single 5xx) should not fail the whole recording into the manual
+// backup-replay path when the very next attempt would land
+const CHUNK_UPLOAD_MAX_ATTEMPTS = 3;
+const CHUNK_UPLOAD_RETRY_BASE_MS = 250;
+
+// Only transient server responses are worth retrying inline; a 4xx (bad
+// request, auth, not found) won't fix itself on the next attempt.
+function isRetriableChunkStatus(status: number): boolean {
+  return status === 429 || status >= 500;
+}
+
 async function uploadChunk(url: string, blob: Blob): Promise<void> {
-  // Signal to the bubble frame pump that a chunk is being uploaded.
-  // The pump's tick loop checks this flag and yields its slot to the
-  // fetch for the ~150-300ms the POST takes to serialize and land.
-  // Cleared in `finally` below so a throw still releases the pump.
-  incChunkBusy();
-  let res: Response;
-  try {
-    res = await fetch(url, {
-      method: "POST",
-      headers: { "Content-Type": blob.type || "application/octet-stream" },
-      // Tauri webview runs on localhost:1420 (dev) or tauri://localhost (prod);
-      // the clips server is a different origin. The framework's dev CORS is
-      // permissive for "*" but won't accept credentialed requests without
-      // Allow-Credentials — and in dev auth is bypassed anyway, so we don't
-      // need cookies.
-      credentials: "include",
-      body: blob,
-    });
-  } finally {
-    decChunkBusy();
-  }
-  if (!res.ok) {
-    const body = await res.text().catch(() => "");
-    console.error(
-      "[clips-recorder] chunk failed:",
-      res.status,
-      body.slice(0, 200),
-    );
-    throw new Error(`chunk ${res.status}: ${body.slice(0, 200)}`);
-  }
-  // Drain the response body even on success. If we don't consume the
-  // body, WebKit can keep the network buffer resident until GC — that's
-  // extra retention on top of the ~1MB Blob we just uploaded. Reading
-  // and discarding is cheap (the body is usually tiny for a chunk ack)
-  // and makes the memory footprint predictable.
-  try {
-    await res.text();
-  } catch {
-    // ignore — body drain is best-effort
+  let lastError: Error | null = null;
+  for (let attempt = 1; attempt <= CHUNK_UPLOAD_MAX_ATTEMPTS; attempt++) {
+    // Signal to the bubble frame pump that a chunk is being uploaded, but only
+    // around the actual network call. The pump's tick loop checks this flag and
+    // yields its slot to the fetch for the ~150-300ms the POST takes to
+    // serialize and land. Released before any backoff wait so the pump can
+    // encode frames while we sit idle between attempts.
+    incChunkBusy();
+    let res: Response | null = null;
+    try {
+      res = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": blob.type || "application/octet-stream" },
+        // Tauri webview runs on localhost:1420 (dev) or tauri://localhost (prod);
+        // the clips server is a different origin. The framework's dev CORS is
+        // permissive for "*" but won't accept credentialed requests without
+        // Allow-Credentials — and in dev auth is bypassed anyway, so we don't
+        // need cookies.
+        credentials: "include",
+        body: blob,
+      });
+    } catch (err) {
+      // Network-level failure (offline, connection reset, DNS) — transient.
+      lastError = err instanceof Error ? err : new Error(String(err));
+    } finally {
+      decChunkBusy();
+    }
+
+    if (res) {
+      if (res.ok) {
+        // Drain the response body even on success. If we don't consume the
+        // body, WebKit can keep the network buffer resident until GC — that's
+        // extra retention on top of the ~1MB Blob we just uploaded. Reading
+        // and discarding is cheap (the body is usually tiny for a chunk ack)
+        // and makes the memory footprint predictable.
+        try {
+          await res.text();
+        } catch {
+          // ignore — body drain is best-effort
+        }
+        console.log(
+          "[clips-recorder] chunk ok:",
+          res.status,
+          blob.size,
+          "bytes",
+        );
+        return;
+      }
+      const body = await res.text().catch(() => "");
+      lastError = new Error(`chunk ${res.status}: ${body.slice(0, 200)}`);
+      if (!isRetriableChunkStatus(res.status)) {
+        console.error(
+          "[clips-recorder] chunk failed:",
+          res.status,
+          body.slice(0, 200),
+        );
+        throw lastError;
+      }
+      console.warn(
+        "[clips-recorder] chunk retriable failure:",
+        res.status,
+        `attempt ${attempt}/${CHUNK_UPLOAD_MAX_ATTEMPTS}`,
+      );
+    } else {
+      console.warn(
+        "[clips-recorder] chunk network error:",
+        lastError?.message,
+        `attempt ${attempt}/${CHUNK_UPLOAD_MAX_ATTEMPTS}`,
+      );
+    }
+
+    if (attempt < CHUNK_UPLOAD_MAX_ATTEMPTS) {
+      await wait(CHUNK_UPLOAD_RETRY_BASE_MS * 2 ** (attempt - 1));
+    }
   }
-  console.log("[clips-recorder] chunk ok:", res.status, blob.size, "bytes");
+  throw lastError ?? new Error("chunk upload failed");
 }
 
 async function abortRecordingUpload(
@@ -2804,6 +2873,17 @@ async function startRecordingInner(
     .forEach((track) => uploadCombined.addTrack(track));
   recordingAudio.tracks.forEach((track) => uploadCombined.addTrack(track));
 
+  // MIME type is resolved up front so create-recording can initialize the
+  // resumable session with the correct content type when the server supports
+  // streaming uploads.
+  const mimeCandidates = [
+    "video/webm;codecs=vp9,opus",
+    "video/webm;codecs=vp8,opus",
+    "video/webm",
+  ];
+  const mimeType =
+    mimeCandidates.find((m) => MediaRecorder.isTypeSupported(m)) ?? "";
+
   // 2+3. Countdown + create-recording happen IN PARALLEL. The countdown is
   // pure visual feedback — gating it on a network round-trip makes the
   // 3-2-1 feel laggy after the user picks a screen. Kick both off and
@@ -2818,6 +2898,7 @@ async function startRecordingInner(
     wantsCamera,
     wantsAudio,
     captureTitle,
+    { mimeType: mimeType || "video/webm", requestStreaming: true },
   ).finally(() => {
     console.timeEnd("[clips-recorder] createServerRecording duration");
   });
@@ -2833,12 +2914,12 @@ async function startRecordingInner(
     );
     throw err;
   }
-  const { id } = createRes;
+  const { id, uploadMode } = createRes;
   console.log(
     "[clips-recorder] countdown + createServerRecording both resolved, id=",
     id,
   );
-  console.log("[clips-recorder] recording row created", { id });
+  console.log("[clips-recorder] recording row created", { id, uploadMode });
   let nativeTranscriptFailureSaved = false;
   const saveTranscriptFailure = async (failureReason: string) => {
     if (!wantsAudio || nativeTranscriptFailureSaved) return;
@@ -2853,17 +2934,15 @@ async function startRecordingInner(
 
   // 4. Start MediaRecorder with a 2-second timeslice — each `ondataavailable`
   //    streams a chunk to the server, so we don't hold 5-min buffers in memory.
-  const mimeCandidates = [
-    "video/webm;codecs=vp9,opus",
-    "video/webm;codecs=vp8,opus",
-    "video/webm",
-  ];
-  const mimeType =
-    mimeCandidates.find((m) => MediaRecorder.isTypeSupported(m)) ?? "";
   const recorder = createCloudMediaRecorder(uploadCombined, mimeType);
   let chunkIndex = 0;
   let failed: Error | null = null;
   let backupBytes = 0;
+  // Backup chunks are indexed by raw MediaRecorder blob (one per
+  // `ondataavailable`), independent of `chunkIndex` — on the streaming path
+  // `chunkIndex` counts aligned upload slices, not raw blobs.
+  let backupChunkCount = 0;
+  const streamMimeType = mimeType || "video/webm";
   let backupMeta: BrowserRecordingBackupMeta = {
     recordingId: id,
     serverUrl: params.serverUrl.replace(/\/+$/, ""),
@@ -2889,6 +2968,41 @@ async function startRecordingInner(
   persistBackupMeta().catch((err) => {
     console.warn("[clips-recorder] local backup metadata failed:", err);
   });
+
+  // Every raw MediaRecorder blob is mirrored to IndexedDB on both upload paths.
+  // If uploads fail the recording can still be recovered locally — replayed to
+  // the server (the retry first resets any resumable session so replay routes
+  // through the buffered chunk path) or exported to a local file.
+  const backupWrites = new Set<Promise<void>>();
+  const backupChunkLocally = (blob: Blob): Promise<void> => {
+    const backupIdx = backupChunkCount++;
+    backupBytes += blob.size;
+    const chunkMimeType = blob.type || streamMimeType;
+    let w: Promise<void>;
+    w = (async () => {
+      try {
+        await putBrowserRecordingBackupChunk({
+          recordingId: id,
+          index: backupIdx,
+          blob,
+          bytes: blob.size,
+          mimeType: chunkMimeType,
+          createdAt: new Date().toISOString(),
+        });
+        await persistBackupMeta({
+          bytes: backupBytes,
+          chunkCount: backupChunkCount,
+          mimeType: chunkMimeType,
+        });
+      } catch (err) {
+        console.warn("[clips-recorder] local chunk backup failed:", err);
+      }
+    })().finally(() => {
+      backupWrites.delete(w);
+    });
+    backupWrites.add(w);
+    return w;
+  };
   // In-flight chunk uploads. We use a Set (not an array) so entries can be
   // removed as soon as each fetch settles — otherwise, for a 30-minute
   // recording the array grows to 900 Promises, and EACH promise closes over
@@ -2898,10 +3012,62 @@ async function startRecordingInner(
   // See `uploadChunk()` — it removes its own entry in `.finally()`.
   const inflight = new Set<Promise<void>>();
 
+  // Streaming-path state. When the server opened a resumable session, blobs
+  // accumulate here until at least STREAM_CHUNK_BYTES is available, then a
+  // 256 KiB-aligned slice is uploaded as a non-final chunk. Resumable sessions
+  // append by byte offset server-side, so streamed chunks MUST arrive in order
+  // — uploads are serialized through `streamQueue`. The unaligned remainder is
+  // sent as the final chunk on stop().
+  let pendingStreamBlobs: Blob[] = [];
+  let pendingStreamBytes = 0;
+  let streamQueue: Promise<void> = Promise.resolve();
+
+  const queueStreamChunk = (blob: Blob, idx: number) => {
+    const url = chunkUrl(params.serverUrl, id, idx, false, {
+      mimeType: streamMimeType,
+    });
+    streamQueue = streamQueue.then(async () => {
+      if (failed) return;
+      try {
+        await uploadChunk(url, blob);
+      } catch (err) {
+        failed ??= err instanceof Error ? err : new Error(String(err));
+      }
+    });
+  };
+
+  const flushAlignedStreamChunks = () => {
+    while (pendingStreamBytes >= STREAM_CHUNK_BYTES) {
+      const combined = new Blob(pendingStreamBlobs, { type: streamMimeType });
+      const head = combined.slice(0, STREAM_CHUNK_BYTES, streamMimeType);
+      const tail = combined.slice(
+        STREAM_CHUNK_BYTES,
+        combined.size,
+        streamMimeType,
+      );
+      pendingStreamBlobs = tail.size > 0 ? [tail] : [];
+      pendingStreamBytes = tail.size;
+      queueStreamChunk(head, chunkIndex++);
+    }
+  };
+
   recorder.ondataavailable = (ev) => {
     if (!ev.data || ev.data.size === 0) return;
+
+    // Always mirror the raw blob to the local backup first (disaster recovery).
+    void backupChunkLocally(ev.data);
+
+    if (uploadMode === "streaming") {
+      // Resumable session on the server: buffer and flush 256 KiB-aligned
+      // slices, uploaded in order. The unaligned remainder is sent as the
+      // final chunk on stop().
+      pendingStreamBlobs.push(ev.data);
+      pendingStreamBytes += ev.data.size;
+      flushAlignedStreamChunks();
+      return;
+    }
+
     const idx = chunkIndex++;
-    backupBytes += ev.data.size;
     const chunkMimeType = ev.data.type || mimeType || "video/webm";
     const url = chunkUrl(params.serverUrl, id, idx, false, {
       mimeType: chunkMimeType,
@@ -2912,26 +3078,7 @@ async function startRecordingInner(
     // constructing the promise body so `inflight.delete(p)` inside the
     // `.finally` can reference the same handle we added.
     let p: Promise<void>;
-    p = (async () => {
-      try {
-        await putBrowserRecordingBackupChunk({
-          recordingId: id,
-          index: idx,
-          blob: ev.data,
-          bytes: ev.data.size,
-          mimeType: chunkMimeType,
-          createdAt: new Date().toISOString(),
-        });
-        await persistBackupMeta({
-          bytes: backupBytes,
-          chunkCount: Math.max(backupMeta.chunkCount, idx + 1),
-          mimeType: chunkMimeType,
-        });
-      } catch (err) {
-        console.warn("[clips-recorder] local chunk backup failed:", err);
-      }
-      await uploadChunk(url, ev.data);
-    })()
+    p = uploadChunk(url, ev.data)
       .catch((err) => {
         failed ??= err instanceof Error ? err : new Error(String(err));
       })
@@ -3182,7 +3329,7 @@ async function startRecordingInner(
         bytes: backupBytes,
         hasAudio: uploadCombined.getAudioTracks().length > 0,
         hasCamera: wantsCamera,
-        chunkCount: chunkIndex,
+        chunkCount: backupChunkCount,
         mimeType: finalMimeType,
         lastError: null,
       }).catch((err) => {
@@ -3235,11 +3382,19 @@ async function startRecordingInner(
 
       // Wait for any in-flight chunk uploads to settle before sending the
       // final chunk. Otherwise the server could finalize before the last
-      // few bytes land. Snapshot the current set — the `.finally` in each
-      // upload will have already removed settled entries from `inflight`.
+      // few bytes land. On the streaming path uploads are serialized through
+      // `streamQueue`; on the buffered path they run concurrently and each
+      // `.finally` has already removed settled entries from `inflight`.
       const pending = Array.from(inflight);
-      await Promise.allSettled(pending);
+      if (uploadMode === "streaming") {
+        await streamQueue;
+      } else {
+        await Promise.allSettled(pending);
+      }
       inflight.clear();
+      // Ensure the local backup is fully written before we either delete it
+      // (on success) or leave it in place for recovery (on failure).
+      await Promise.allSettled([...backupWrites]);
       if (failed) {
         console.error("[clips-recorder] chunk upload failed:", failed);
         await markBrowserRecordingBackupError(id, failed.message).catch(
@@ -3253,6 +3408,17 @@ async function startRecordingInner(
         throw failed;
       }
 
+      // Streaming: the closing bytes are whatever remains under the 256 KiB
+      // alignment boundary — send them as the final chunk so the resumable
+      // session can complete. Buffered: bytes are already staged server-side,
+      // so the final chunk is a 0-byte close sentinel.
+      const finalBody =
+        uploadMode === "streaming"
+          ? new Blob(pendingStreamBlobs, { type: finalMimeType })
+          : new Blob([], { type: finalMimeType });
+      pendingStreamBlobs = [];
+      pendingStreamBytes = 0;
+
       const finalizeUrl = chunkUrl(params.serverUrl, id, chunkIndex, true, {
         mimeType: finalMimeType,
         durationMs: String(durationMs),
@@ -3264,6 +3430,8 @@ async function startRecordingInner(
       console.log("[clips-recorder] finalize POST", finalizeUrl, {
         chunksSent: chunkIndex,
         inflightAtFinalize: pending.length,
+        finalBodyBytes: finalBody.size,
+        uploadMode,
         anyFailed: !!failed,
       });
       try {
@@ -3271,7 +3439,7 @@ async function startRecordingInner(
           method: "POST",
           headers: { "Content-Type": "application/octet-stream" },
           credentials: "include",
-          body: new Blob([], { type: finalMimeType }),
+          body: finalBody,
         });
         const bodyText = await finalRes.text().catch(() => "");
         console.log(