@agent-native/core 0.70.3 → 0.71.0

package/corpus/templates/clips/chrome-extension/src/offscreen.ts

@@ -3,11 +3,11 @@
 // This document holds the getDisplayMedia()/getUserMedia() stream and the
 // MediaRecorder. Living in an offscreen document (reason DISPLAY_MEDIA) is what
 // lets a recording survive page navigations — the capture is decoupled from any
-// tab. The camera bubble and controls are rendered as on-page overlays by the
-// content script, so we no longer composite the camera into a canvas here; for
-// screen modes we record the display stream directly (+ mixed mic audio), and
-// the bubble shows up in the recording naturally when a full screen/window/tab
-// surface that contains it is captured.
+// tab. Controls render as on-page overlays. For SCREEN+CAMERA we capture the
+// camera HERE and composite it (canvas) into the recording, because the on-page
+// bubble can't get the camera on pages that send `Permissions-Policy: camera=()`
+// (an iframe can't escape its parent's policy). Screen-only records the display
+// directly; camera-only records the webcam directly.
 //
 // Lifecycle: ACQUIRE (show picker, hold stream) → BEGIN (start recorder after
 // the countdown) → PAUSE/RESUME → STOP/CANCEL, plus RESTART (discard and start
@@ -21,6 +21,9 @@ type AcquireMessage = {
   mode: CaptureMode;
   surface: "browser" | "window" | "monitor";
   includeMicrophone: boolean;
+  // Screen+camera: capture the camera here and composite it into the recording
+  // (the on-page bubble can be blocked by the page's Permissions-Policy).
+  includeCamera: boolean;
 };
 
 type BeginMessage = {
@@ -94,6 +97,13 @@ type ActiveRecording = {
   restarting: boolean;
   // Pending pre-roll timer; non-null means the recorder hasn't started yet.
   startTimer: ReturnType<typeof setTimeout> | null;
+  // Canvas compositor draw loop (screen+camera only); stop it on teardown.
+  stopCompositor: (() => void) | null;
+  // Original capture streams, kept distinct so restart can re-home exactly the
+  // right ones (sourceStreams also holds the derived canvas stream).
+  displaySource: MediaStream | null;
+  cameraSource: MediaStream | null;
+  micSource: MediaStream | null;
   dimensions: { width: number; height: number };
   hasAudio: boolean;
   hasCamera: boolean;
@@ -194,22 +204,140 @@ function displayConstraints(
   } as MediaStreamConstraints;
 }
 
-async function getMicStream(): Promise<MediaStream | null> {
+// The user's chosen camera/mic devices (set in the popup, saved to storage).
+async function readDeviceIds(): Promise<{ video: string; audio: string }> {
   try {
-    return await navigator.mediaDevices.getUserMedia({
-      audio: {
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true,
-        channelCount: 1,
-      },
-      video: false,
-    });
+    const v = await chrome.storage.sync.get(["videoDeviceId", "audioDeviceId"]);
+    return {
+      video: typeof v.videoDeviceId === "string" ? v.videoDeviceId : "",
+      audio: typeof v.audioDeviceId === "string" ? v.audioDeviceId : "",
+    };
+  } catch {
+    return { video: "", audio: "" };
+  }
+}
+
+function cameraConstraints(deviceId: string): MediaTrackConstraints {
+  const base: MediaTrackConstraints = {
+    width: { ideal: 1280 },
+    height: { ideal: 720 },
+  };
+  if (deviceId) base.deviceId = { exact: deviceId };
+  else base.facingMode = "user";
+  return base;
+}
+
+async function getMicStream(deviceId: string): Promise<MediaStream | null> {
+  const audio: MediaTrackConstraints = {
+    echoCancellation: true,
+    noiseSuppression: true,
+    autoGainControl: true,
+    channelCount: 1,
+  };
+  if (deviceId) audio.deviceId = { exact: deviceId };
+  try {
+    return await navigator.mediaDevices.getUserMedia({ audio, video: false });
   } catch {
     return null;
   }
 }
 
+/* ----------------------------------------------------- camera compositing --- */
+
+// On many pages the on-page camera bubble can't run (the page sets
+// `Permissions-Policy: camera=()`, which an iframe cannot escape). So for
+// screen+camera we capture the camera HERE in the offscreen document (extension
+// origin — always allowed) and draw it into a canvas on top of the screen, then
+// record the canvas. The face ends up in the video on every page.
+
+async function readyVideo(stream: MediaStream): Promise<HTMLVideoElement> {
+  const video = document.createElement("video");
+  video.muted = true;
+  video.playsInline = true;
+  video.srcObject = stream;
+  await waitForMetadata(video).catch(() => undefined);
+  await video.play().catch(() => undefined);
+  return video;
+}
+
+function drawCameraBubble(
+  ctx: CanvasRenderingContext2D,
+  camera: HTMLVideoElement,
+  canvas: HTMLCanvasElement,
+): void {
+  const size = Math.round(Math.min(canvas.width, canvas.height) * 0.22);
+  const margin = Math.round(Math.min(canvas.width, canvas.height) * 0.035);
+  const cx = margin + size / 2;
+  const cy = canvas.height - margin - size / 2;
+  const r = size / 2;
+  const vw = camera.videoWidth || size;
+  const vh = camera.videoHeight || size;
+  const scale = Math.max(size / vw, size / vh);
+  const dw = vw * scale;
+  const dh = vh * scale;
+
+  ctx.save();
+  ctx.beginPath();
+  ctx.arc(cx, cy, r, 0, Math.PI * 2);
+  ctx.closePath();
+  ctx.clip();
+  // Mirror the camera horizontally to match how people expect to see themselves.
+  ctx.translate(cx, cy);
+  ctx.scale(-1, 1);
+  ctx.drawImage(camera, -dw / 2, -dh / 2, dw, dh);
+  ctx.restore();
+
+  ctx.save();
+  ctx.beginPath();
+  ctx.arc(cx, cy, r - 1, 0, Math.PI * 2);
+  ctx.lineWidth = Math.max(3, Math.round(size * 0.03));
+  ctx.strokeStyle = "rgba(255,255,255,0.92)";
+  ctx.stroke();
+  ctx.restore();
+}
+
+type Compositor = {
+  videoTrack: MediaStreamTrack;
+  canvasStream: MediaStream;
+  stop: () => void;
+};
+
+async function buildCompositor(
+  displayStream: MediaStream,
+  cameraStream: MediaStream,
+  dims: { width: number; height: number },
+): Promise<Compositor> {
+  const displayVideo = await readyVideo(displayStream);
+  const cameraVideo = await readyVideo(cameraStream);
+  const canvas = document.createElement("canvas");
+  canvas.width = dims.width;
+  canvas.height = dims.height;
+  const ctx = canvas.getContext("2d");
+  if (!ctx) throw new Error("Canvas 2D context unavailable for compositing.");
+
+  let running = true;
+  let raf = 0;
+  const draw = () => {
+    if (!running) return;
+    ctx.drawImage(displayVideo, 0, 0, canvas.width, canvas.height);
+    if (cameraVideo.readyState >= 2) drawCameraBubble(ctx, cameraVideo, canvas);
+    raf = requestAnimationFrame(draw);
+  };
+  raf = requestAnimationFrame(draw);
+
+  const canvasStream = canvas.captureStream(30);
+  return {
+    videoTrack: canvasStream.getVideoTracks()[0],
+    canvasStream,
+    stop: () => {
+      running = false;
+      cancelAnimationFrame(raf);
+      displayVideo.srcObject = null;
+      cameraVideo.srcObject = null;
+    },
+  };
+}
+
 async function createMixedAudio(
   streams: MediaStream[],
 ): Promise<{ audioContext: AudioContext | null; tracks: MediaStreamTrack[] }> {
@@ -287,6 +415,13 @@ async function uploadChunk(
   const text = await res.text().catch(() => "");
   const data = text ? (JSON.parse(text) as UploadResult) : {};
   if (!res.ok) {
+    console.error(
+      "[clips-offscreen] chunk upload failed",
+      res.status,
+      "hadAuth:",
+      Boolean(recording.authToken),
+      text.slice(0, 200),
+    );
     throw new Error(
       data?.error || `Upload failed (${res.status}): ${text || res.statusText}`,
     );
@@ -310,6 +445,7 @@ function disposePrepared(): void {
 }
 
 function cleanup(recording: ActiveRecording): void {
+  recording.stopCompositor?.();
   stopStreams([recording.outputStream, ...recording.sourceStreams]);
   void recording.audioContext?.close().catch(() => undefined);
 }
@@ -329,22 +465,29 @@ async function acquire(message: AcquireMessage): Promise<{
   let displayStream: MediaStream | null = null;
   let micStream: MediaStream | null = null;
   let cameraStream: MediaStream | null = null;
+  const devices = await readDeviceIds();
 
   if (message.mode === "camera") {
     cameraStream = await navigator.mediaDevices.getUserMedia({
-      video: {
-        width: { ideal: 1280 },
-        height: { ideal: 720 },
-        facingMode: "user",
-      },
-      audio: message.includeMicrophone,
+      video: cameraConstraints(devices.video),
+      audio: message.includeMicrophone
+        ? devices.audio
+          ? { deviceId: { exact: devices.audio } }
+          : true
+        : false,
     });
   } else {
     // Native "Choose what to share" picker. This is the screenshot Steve showed.
     displayStream = await navigator.mediaDevices.getDisplayMedia(
       displayConstraints(message.surface),
     );
-    if (message.includeMicrophone) micStream = await getMicStream();
+    if (message.includeMicrophone)
+      micStream = await getMicStream(devices.audio);
+    // The screen+camera face comes from the on-page bubble (captured in the
+    // display pixels), NOT composited here: canvas/requestAnimationFrame does
+    // not run in a hidden offscreen document, so compositing produced an empty
+    // recording ("No chunks found"). We record the display stream directly.
+    void message.includeCamera;
   }
 
   const videoStream = displayStream ?? cameraStream;
@@ -401,8 +544,24 @@ async function begin(message: BeginMessage): Promise<{
   if (activeRecording) throw new Error("Clips is already recording.");
 
   const videoStream = ready.displayStream ?? ready.cameraStream;
-  const videoTrack = videoStream?.getVideoTracks()[0];
-  if (!videoTrack) throw new Error("Capture video track was lost.");
+  const directVideoTrack = videoStream?.getVideoTracks()[0];
+  if (!directVideoTrack) throw new Error("Capture video track was lost.");
+
+  // Screen + camera → composite the camera bubble into a canvas and record that,
+  // so the face is in the video even on pages that block the on-page bubble.
+  let compositor: Compositor | null = null;
+  let videoTrack = directVideoTrack;
+  if (ready.displayStream && ready.cameraStream) {
+    compositor = await buildCompositor(
+      ready.displayStream,
+      ready.cameraStream,
+      {
+        width: ready.width,
+        height: ready.height,
+      },
+    );
+    videoTrack = compositor.videoTrack;
+  }
 
   const audioInputs = [
     ...(ready.displayStream ? [ready.displayStream] : []),
@@ -444,6 +603,7 @@ async function begin(message: BeginMessage): Promise<{
       ...(ready.displayStream ? [ready.displayStream] : []),
       ...(ready.micStream ? [ready.micStream] : []),
       ...(ready.cameraStream ? [ready.cameraStream] : []),
+      ...(compositor ? [compositor.canvasStream] : []),
     ],
     audioContext: mixedAudio.audioContext,
     chunkIndex: 0,
@@ -452,6 +612,10 @@ async function begin(message: BeginMessage): Promise<{
     cancelled: false,
     restarting: false,
     startTimer: null,
+    stopCompositor: compositor?.stop ?? null,
+    displaySource: ready.displayStream,
+    cameraSource: ready.cameraStream,
+    micSource: ready.micStream,
     dimensions: { width: ready.width, height: ready.height },
     hasAudio: outputStream.getAudioTracks().length > 0,
     hasCamera:
@@ -518,6 +682,7 @@ function startRecorderNow(recording: ActiveRecording): void {
   try {
     recording.recorder.start(2000);
     recording.startedAtMs = Date.now();
+    console.log("[clips-offscreen] recorder.start ok");
     reportStatus(recording.sessionId, "recording", {
       recordingId: recording.recordingId,
       width: recording.dimensions.width,
@@ -685,22 +850,18 @@ async function restart(
   recording.restarting = true;
   recording.cancelled = true;
   if (recording.recorder.state !== "inactive") recording.recorder.stop();
-  // Close only the old mixing context; keep the capture tracks alive.
+  // Tear down the old compositor + mixing context; keep the capture tracks alive
+  // so the next BEGIN can build a fresh compositor/recorder on the same streams.
+  recording.stopCompositor?.();
   void recording.audioContext?.close().catch(() => undefined);
   activeRecording = null;
 
-  const videoStream =
-    recording.sourceStreams.find((s) => s.getVideoTracks().length) ?? null;
-  const isCamera = recording.mode === "camera";
   prepared = {
     sessionId: recording.sessionId,
     mode: recording.mode,
-    displayStream: isCamera ? null : videoStream,
-    cameraStream: isCamera ? videoStream : null,
-    micStream:
-      recording.sourceStreams.find(
-        (s) => s.getAudioTracks().length && s !== videoStream,
-      ) ?? null,
+    displayStream: recording.displaySource,
+    cameraStream: recording.cameraSource,
+    micStream: recording.micSource,
     width: recording.dimensions.width,
     height: recording.dimensions.height,
     endedListener: null,

package/corpus/templates/clips/chrome-extension/src/overlay.css

@@ -39,10 +39,9 @@ body {
   height: 100%;
   border-radius: 50%;
   overflow: hidden;
-  background: #0f172a;
-  box-shadow:
-    0 10px 30px rgba(15, 23, 42, 0.35),
-    0 0 0 4px rgba(255, 255, 255, 0.92);
+  background: #18181b;
+  border: 3px solid rgba(255, 255, 255, 0.92);
+  box-shadow: 0 6px 18px rgba(9, 9, 11, 0.4);
   display: flex;
   align-items: center;
   justify-content: center;
@@ -75,7 +74,7 @@ body {
   display: flex;
   align-items: center;
   justify-content: center;
-  background: rgba(2, 6, 23, 0.55);
+  background: rgba(9, 9, 11, 0.55);
   backdrop-filter: blur(2px);
 }
 
@@ -126,9 +125,9 @@ body {
   font-size: clamp(96px, 22vmin, 240px);
   font-weight: 700;
   color: #fff;
-  text-shadow: 0 6px 30px rgba(2, 6, 23, 0.6);
+  text-shadow: 0 6px 30px rgba(9, 9, 11, 0.6);
   line-height: 1;
-  animation: countdown-pop 1s ease-out forwards;
+  animation: countdown-pop 0.45s ease-out forwards;
 }
 
 .countdown-go {
@@ -138,21 +137,17 @@ body {
 
 @keyframes countdown-pop {
   0% {
-    transform: scale(0.45);
+    transform: scale(0.55);
     opacity: 0;
   }
-  25% {
-    transform: scale(1);
+  45% {
+    transform: scale(1.08);
     opacity: 1;
   }
-  70% {
+  100% {
     transform: scale(1);
     opacity: 1;
   }
-  100% {
-    transform: scale(1.25);
-    opacity: 0;
-  }
 }
 
 .countdown-hint {
@@ -163,7 +158,7 @@ body {
   color: rgba(255, 255, 255, 0.85);
   font-size: 15px;
   font-weight: 500;
-  background: rgba(2, 6, 23, 0.45);
+  background: rgba(9, 9, 11, 0.45);
   padding: 8px 16px;
   border-radius: 999px;
 }
@@ -181,19 +176,21 @@ body {
 }
 
 .toolbar-v {
+  /* Top-anchored so Stop/time/pause stay put and the pill grows DOWNWARD on
+     hover (centering shifted the buttons up and caused misclicks). */
   position: absolute;
-  top: 50%;
+  top: 0;
   left: 50%;
-  transform: translate(-50%, -50%);
+  transform: translateX(-50%);
   display: flex;
   flex-direction: column;
   align-items: center;
   gap: 8px;
   padding: 9px;
   border-radius: 20px;
-  background: rgba(15, 23, 42, 0.94);
+  background: rgba(24, 24, 27, 0.94);
   border: 1px solid rgba(255, 255, 255, 0.1);
-  box-shadow: 0 10px 28px rgba(2, 6, 23, 0.45);
+  box-shadow: 0 10px 28px rgba(9, 9, 11, 0.45);
   color: #fff;
   -webkit-user-select: none;
   user-select: none;
@@ -323,9 +320,9 @@ body {
   gap: 9px;
   padding: 14px 16px;
   border-radius: 14px;
-  background: rgba(15, 23, 42, 0.95);
+  background: rgba(24, 24, 27, 0.95);
   border: 1px solid rgba(255, 255, 255, 0.1);
-  box-shadow: 0 10px 28px rgba(2, 6, 23, 0.45);
+  box-shadow: 0 10px 28px rgba(9, 9, 11, 0.45);
   color: #fff;
 }
 
@@ -363,7 +360,7 @@ body {
   height: 100%;
   width: 40%;
   border-radius: 999px;
-  background: #6366f1;
+  background: #d4d4d8;
   animation: saving-indeterminate 1.1s ease-in-out infinite;
 }
 
@@ -375,3 +372,56 @@ body {
     margin-left: 100%;
   }
 }
+
+/* Bubble size dots (small / large), revealed on hover — desktop-style. */
+.bubble-sizes {
+  position: absolute;
+  bottom: 7%;
+  left: 50%;
+  transform: translateX(-50%);
+  display: flex;
+  align-items: center;
+  gap: 9px;
+  padding: 5px 9px;
+  border-radius: 999px;
+  background: rgba(9, 9, 11, 0.55);
+  -webkit-backdrop-filter: blur(4px);
+  backdrop-filter: blur(4px);
+  opacity: 0;
+  transition: opacity 0.15s ease;
+  pointer-events: none;
+}
+
+.bubble:hover .bubble-sizes {
+  opacity: 1;
+  pointer-events: auto;
+}
+
+.bubble-size-dot {
+  border: none;
+  padding: 0;
+  border-radius: 50%;
+  background: rgba(255, 255, 255, 0.55);
+  cursor: pointer;
+  transition:
+    background 0.12s ease,
+    transform 0.1s ease;
+}
+
+.bubble-size-dot:hover {
+  background: #fff;
+}
+
+.bubble-size-dot:active {
+  transform: scale(0.9);
+}
+
+.bubble-size-sm {
+  width: 8px;
+  height: 8px;
+}
+
+.bubble-size-lg {
+  width: 13px;
+  height: 13px;
+}

package/corpus/templates/clips/chrome-extension/src/overlay.ts

@@ -75,21 +75,82 @@ function send(type: string, extra: Record<string, unknown> = {}): void {
 
 /* ---------------------------------------------------------------- bubble --- */
 
+function postBubble(kind: string, extra: Record<string, unknown> = {}): void {
+  try {
+    window.parent.postMessage(
+      { source: "clips-overlay", kind, part: "bubble", ...extra },
+      "*",
+    );
+  } catch {
+    /* parent gone */
+  }
+}
+
 async function initBubble(): Promise<void> {
   const bubble = document.createElement("div");
   bubble.className = "bubble";
   const ring = document.createElement("div");
   ring.className = "bubble-ring";
   bubble.appendChild(ring);
+
+  // Drag: the content script owns the iframe position, so we just signal the
+  // start of a drag and it captures the pointer page-wide.
+  bubble.style.cursor = "grab";
+  bubble.addEventListener("pointerdown", (e) => {
+    if (e.button !== 0) return;
+    if ((e.target as HTMLElement).closest("[data-no-drag]")) return;
+    e.preventDefault();
+    bubble.style.cursor = "grabbing";
+    postBubble("bubble-drag-start");
+    const restore = (): void => {
+      bubble.style.cursor = "grab";
+      window.removeEventListener("pointerup", restore);
+    };
+    window.addEventListener("pointerup", restore);
+  });
+
+  // Size dots (small / large), revealed on hover — like the desktop bubble.
+  const sizes = document.createElement("div");
+  sizes.className = "bubble-sizes";
+  sizes.setAttribute("data-no-drag", "");
+  for (const key of ["sm", "lg"] as const) {
+    const dot = document.createElement("button");
+    dot.type = "button";
+    dot.className = `bubble-size-dot bubble-size-${key}`;
+    dot.title = key === "sm" ? "Small" : "Large";
+    dot.setAttribute(
+      "aria-label",
+      key === "sm" ? "Small bubble" : "Large bubble",
+    );
+    dot.setAttribute("data-no-drag", "");
+    dot.addEventListener("click", (e) => {
+      e.stopPropagation();
+      postBubble("bubble-size", { size: key });
+    });
+    sizes.appendChild(dot);
+  }
+  bubble.appendChild(sizes);
+
   root.appendChild(bubble);
 
   try {
+    const videoDeviceId = await new Promise<string>((resolve) => {
+      try {
+        chrome.storage.sync.get("videoDeviceId", (v) =>
+          resolve(typeof v.videoDeviceId === "string" ? v.videoDeviceId : ""),
+        );
+      } catch {
+        resolve("");
+      }
+    });
+    const videoConstraint: MediaTrackConstraints = {
+      width: { ideal: 640 },
+      height: { ideal: 640 },
+    };
+    if (videoDeviceId) videoConstraint.deviceId = { exact: videoDeviceId };
+    else videoConstraint.facingMode = "user";
     const stream = await navigator.mediaDevices.getUserMedia({
-      video: {
-        width: { ideal: 640 },
-        height: { ideal: 640 },
-        facingMode: "user",
-      },
+      video: videoConstraint,
       audio: false,
     });
     const video = document.createElement("video");
@@ -99,7 +160,9 @@ async function initBubble(): Promise<void> {
     video.srcObject = stream;
     ring.appendChild(video);
     await video.play().catch(() => undefined);
-  } catch {
+    console.log("[clips-overlay] camera bubble live");
+  } catch (err) {
+    console.warn("[clips-overlay] camera getUserMedia failed:", err);
     const empty = document.createElement("div");
     empty.className = "bubble-empty";
     empty.innerHTML = ICONS.cameraOff;
@@ -148,14 +211,20 @@ function initCountdown(): void {
   root.appendChild(wrap);
 
   let lastShown = "";
+  // Compute the fallback end-time ONCE (if the worker's countdownEndsAtMs never
+  // arrives) — recomputing it every frame froze the number on "3".
+  let fallbackEndsAt = 0;
   const render = (): void => {
-    const fallback =
-      Number(params.get("seconds") || String(COUNTDOWN_FALLBACK)) ||
-      COUNTDOWN_FALLBACK;
-    const endsAt =
-      state.countdownEndsAtMs > 0
-        ? state.countdownEndsAtMs
-        : Date.now() + fallback * 1000;
+    let endsAt = state.countdownEndsAtMs;
+    if (endsAt <= 0) {
+      if (fallbackEndsAt === 0) {
+        const fallback =
+          Number(params.get("seconds") || String(COUNTDOWN_FALLBACK)) ||
+          COUNTDOWN_FALLBACK;
+        fallbackEndsAt = Date.now() + fallback * 1000;
+      }
+      endsAt = fallbackEndsAt;
+    }
     const remainingMs = endsAt - Date.now();
     if (state.phase !== "countdown" || remainingMs <= 0) {
       if (lastShown !== "Go") {