@agent-native/core 0.70.2 → 0.71.0

package/corpus/templates/clips/chrome-extension/src/content-script.ts

@@ -13,6 +13,128 @@
   const ALL_PARTS: OverlayPart[] = ["bubble", "countdown", "toolbar", "saving"];
   const flags = window as unknown as { __clipsOverlayHostReady?: boolean };
 
+  // ----- Draggable, resizable camera bubble ---------------------------------
+  // Size + position persist in storage so the bubble stays where the user put it
+  // across pages and recordings (like the desktop app). The iframe can't move or
+  // resize itself, so the content script owns its geometry.
+  const BUBBLE_SIZES: Record<string, number> = { sm: 184, lg: 280 };
+  const bubbleGeom: { size: string; left: number | null; top: number | null } =
+    { size: "lg", left: null, top: null };
+  let bubbleDragLayer: HTMLDivElement | null = null;
+  let bubblePersistTimer: ReturnType<typeof setTimeout> | undefined;
+
+  function bubbleSizePx(): number {
+    return BUBBLE_SIZES[bubbleGeom.size] ?? BUBBLE_SIZES.lg;
+  }
+
+  function clampBubble(
+    left: number,
+    top: number,
+    size: number,
+  ): { left: number; top: number } {
+    return {
+      left: Math.max(8, Math.min(left, window.innerWidth - size - 8)),
+      top: Math.max(8, Math.min(top, window.innerHeight - size - 8)),
+    };
+  }
+
+  function applyBubbleGeom(): void {
+    const frame = document.getElementById(
+      partFrameId("bubble"),
+    ) as HTMLIFrameElement | null;
+    if (!frame) return;
+    const size = bubbleSizePx();
+    const margin = 24;
+    const base = clampBubble(
+      bubbleGeom.left ?? margin,
+      bubbleGeom.top ?? window.innerHeight - size - margin,
+      size,
+    );
+    Object.assign(frame.style, {
+      left: `${base.left}px`,
+      top: `${base.top}px`,
+      bottom: "auto",
+      width: `${size}px`,
+      height: `${size}px`,
+    });
+  }
+
+  function persistBubbleGeom(): void {
+    clearTimeout(bubblePersistTimer);
+    bubblePersistTimer = setTimeout(() => {
+      try {
+        chrome.storage.local.set({ bubbleGeom });
+      } catch {
+        /* ignore */
+      }
+    }, 200);
+  }
+
+  function startBubbleDrag(): void {
+    if (bubbleDragLayer) return;
+    const frame = document.getElementById(
+      partFrameId("bubble"),
+    ) as HTMLIFrameElement | null;
+    if (!frame) return;
+    // Full-screen capture layer so the pointer keeps tracking after it leaves
+    // the small bubble iframe.
+    const layer = document.createElement("div");
+    Object.assign(layer.style, {
+      position: "fixed",
+      inset: "0",
+      zIndex: "2147483647",
+      cursor: "grabbing",
+    });
+    (document.documentElement || document.body).appendChild(layer);
+    bubbleDragLayer = layer;
+    const onMove = (e: PointerEvent): void => {
+      const size = bubbleSizePx();
+      const rect = frame.getBoundingClientRect();
+      const next = clampBubble(
+        rect.left + e.movementX,
+        rect.top + e.movementY,
+        size,
+      );
+      bubbleGeom.left = next.left;
+      bubbleGeom.top = next.top;
+      Object.assign(frame.style, {
+        left: `${next.left}px`,
+        top: `${next.top}px`,
+        bottom: "auto",
+      });
+    };
+    const onUp = (): void => {
+      layer.removeEventListener("pointermove", onMove);
+      layer.removeEventListener("pointerup", onUp);
+      layer.removeEventListener("pointercancel", onUp);
+      layer.remove();
+      bubbleDragLayer = null;
+      persistBubbleGeom();
+    };
+    layer.addEventListener("pointermove", onMove);
+    layer.addEventListener("pointerup", onUp);
+    layer.addEventListener("pointercancel", onUp);
+  }
+
+  try {
+    chrome.storage.local.get("bubbleGeom", (value) => {
+      if (chrome.runtime.lastError) return;
+      const g = value.bubbleGeom as
+        | { size?: unknown; left?: unknown; top?: unknown }
+        | undefined;
+      if (g && typeof g === "object") {
+        bubbleGeom.size = g.size === "sm" ? "sm" : "lg";
+        bubbleGeom.left = typeof g.left === "number" ? g.left : null;
+        bubbleGeom.top = typeof g.top === "number" ? g.top : null;
+      }
+      applyBubbleGeom();
+    });
+  } catch {
+    /* ignore */
+  }
+
+  window.addEventListener("resize", () => applyBubbleGeom());
+
   function requestState(): void {
     try {
       chrome.runtime.sendMessage(
@@ -78,21 +200,25 @@
     frame.setAttribute("allowtransparency", "true");
     if (part === "bubble") {
       frame.allow = "camera; microphone";
+      // Above the countdown so the face stays sharp over the dim/blur. Exact
+      // size/position are set by applyBubbleGeom() once mounted.
+      const size = bubbleSizePx();
       Object.assign(frame.style, {
         left: "24px",
         bottom: "24px",
-        width: "148px",
-        height: "148px",
+        width: `${size}px`,
+        height: `${size}px`,
+        zIndex: "3",
       });
     } else if (part === "toolbar") {
       // Left-edge vertical pill (desktop layout). Height grows on hover via the
       // resize message below.
       Object.assign(frame.style, {
         left: "16px",
-        top: "50%",
-        transform: "translateY(-50%)",
+        top: "calc(50% - 77px)",
         width: "68px",
         height: "154px",
+        zIndex: "2",
       });
     } else if (part === "saving") {
       Object.assign(frame.style, {
@@ -100,12 +226,15 @@
         bottom: "24px",
         width: "264px",
         height: "96px",
+        zIndex: "2",
       });
     } else {
+      // countdown — full-screen dim/blur, below the bubble.
       Object.assign(frame.style, {
         inset: "0",
         width: "100%",
         height: "100%",
+        zIndex: "1",
       });
     }
   }
@@ -114,15 +243,18 @@
     if (document.getElementById(partFrameId(part))) return;
     const frame = document.createElement("iframe");
     frame.id = partFrameId(part);
+    if (part === "bubble") frame.allow = "camera; microphone";
     const url = new URL(chrome.runtime.getURL("src/overlay.html"));
     url.searchParams.set("part", part);
     if (part === "countdown") url.searchParams.set("seconds", "3");
     frame.src = url.toString();
     styleFrame(frame, part);
     container.appendChild(frame);
+    if (part === "bubble") applyBubbleGeom();
   }
 
   function reconcile(parts: OverlayPart[]): void {
+    console.log("[clips-cs] reconcile parts:", parts, "on", location.href);
     const wanted = new Set(parts.filter((p) => ALL_PARTS.includes(p)));
     if (wanted.size === 0) {
       document.getElementById(CONTAINER_ID)?.remove();
@@ -157,20 +289,38 @@
     }
   });
 
-  // The toolbar overlay asks to grow/shrink its own iframe on hover (it can't
-  // resize itself). Only trust messages from our own extension-origin frames.
+  // Overlay iframes post layout requests (toolbar hover-resize, bubble drag and
+  // size). Only trust messages from our own extension-origin frames.
   window.addEventListener("message", (event) => {
     const data = event.data as
-      | { source?: string; kind?: string; part?: string; height?: number }
+      | {
+          source?: string;
+          kind?: string;
+          part?: string;
+          height?: number;
+          size?: string;
+        }
       | undefined;
-    if (!data || data.source !== "clips-overlay" || data.kind !== "resize") {
+    if (!data || data.source !== "clips-overlay") return;
+    if (event.origin !== chrome.runtime.getURL("").replace(/\/$/, "")) return;
+
+    if (data.kind === "resize" && data.part === "toolbar") {
+      const frame = document.getElementById(partFrameId("toolbar"));
+      if (frame && typeof data.height === "number") {
+        frame.style.height = `${Math.round(data.height)}px`;
+      }
+      return;
+    }
+    if (data.kind === "bubble-drag-start") {
+      startBubbleDrag();
+      return;
+    }
+    if (data.kind === "bubble-size") {
+      bubbleGeom.size = data.size === "sm" ? "sm" : "lg";
+      applyBubbleGeom();
+      persistBubbleGeom();
       return;
     }
-    if (event.origin !== chrome.runtime.getURL("").replace(/\/$/, "")) return;
-    const part = data.part === "toolbar" ? "toolbar" : null;
-    if (!part || typeof data.height !== "number") return;
-    const frame = document.getElementById(partFrameId(part));
-    if (frame) frame.style.height = `${Math.round(data.height)}px`;
   });
 
   syncIfRecording();

package/corpus/templates/clips/chrome-extension/src/offscreen.ts

@@ -3,11 +3,11 @@
 // This document holds the getDisplayMedia()/getUserMedia() stream and the
 // MediaRecorder. Living in an offscreen document (reason DISPLAY_MEDIA) is what
 // lets a recording survive page navigations — the capture is decoupled from any
-// tab. The camera bubble and controls are rendered as on-page overlays by the
-// content script, so we no longer composite the camera into a canvas here; for
-// screen modes we record the display stream directly (+ mixed mic audio), and
-// the bubble shows up in the recording naturally when a full screen/window/tab
-// surface that contains it is captured.
+// tab. Controls render as on-page overlays. For SCREEN+CAMERA we capture the
+// camera HERE and composite it (canvas) into the recording, because the on-page
+// bubble can't get the camera on pages that send `Permissions-Policy: camera=()`
+// (an iframe can't escape its parent's policy). Screen-only records the display
+// directly; camera-only records the webcam directly.
 //
 // Lifecycle: ACQUIRE (show picker, hold stream) → BEGIN (start recorder after
 // the countdown) → PAUSE/RESUME → STOP/CANCEL, plus RESTART (discard and start
@@ -21,6 +21,9 @@ type AcquireMessage = {
   mode: CaptureMode;
   surface: "browser" | "window" | "monitor";
   includeMicrophone: boolean;
+  // Screen+camera: capture the camera here and composite it into the recording
+  // (the on-page bubble can be blocked by the page's Permissions-Policy).
+  includeCamera: boolean;
 };
 
 type BeginMessage = {
@@ -94,6 +97,13 @@ type ActiveRecording = {
   restarting: boolean;
   // Pending pre-roll timer; non-null means the recorder hasn't started yet.
   startTimer: ReturnType<typeof setTimeout> | null;
+  // Canvas compositor draw loop (screen+camera only); stop it on teardown.
+  stopCompositor: (() => void) | null;
+  // Original capture streams, kept distinct so restart can re-home exactly the
+  // right ones (sourceStreams also holds the derived canvas stream).
+  displaySource: MediaStream | null;
+  cameraSource: MediaStream | null;
+  micSource: MediaStream | null;
   dimensions: { width: number; height: number };
   hasAudio: boolean;
   hasCamera: boolean;
@@ -194,22 +204,140 @@ function displayConstraints(
   } as MediaStreamConstraints;
 }
 
-async function getMicStream(): Promise<MediaStream | null> {
+// The user's chosen camera/mic devices (set in the popup, saved to storage).
+async function readDeviceIds(): Promise<{ video: string; audio: string }> {
   try {
-    return await navigator.mediaDevices.getUserMedia({
-      audio: {
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true,
-        channelCount: 1,
-      },
-      video: false,
-    });
+    const v = await chrome.storage.sync.get(["videoDeviceId", "audioDeviceId"]);
+    return {
+      video: typeof v.videoDeviceId === "string" ? v.videoDeviceId : "",
+      audio: typeof v.audioDeviceId === "string" ? v.audioDeviceId : "",
+    };
+  } catch {
+    return { video: "", audio: "" };
+  }
+}
+
+function cameraConstraints(deviceId: string): MediaTrackConstraints {
+  const base: MediaTrackConstraints = {
+    width: { ideal: 1280 },
+    height: { ideal: 720 },
+  };
+  if (deviceId) base.deviceId = { exact: deviceId };
+  else base.facingMode = "user";
+  return base;
+}
+
+async function getMicStream(deviceId: string): Promise<MediaStream | null> {
+  const audio: MediaTrackConstraints = {
+    echoCancellation: true,
+    noiseSuppression: true,
+    autoGainControl: true,
+    channelCount: 1,
+  };
+  if (deviceId) audio.deviceId = { exact: deviceId };
+  try {
+    return await navigator.mediaDevices.getUserMedia({ audio, video: false });
   } catch {
     return null;
   }
 }
 
+/* ----------------------------------------------------- camera compositing --- */
+
+// On many pages the on-page camera bubble can't run (the page sets
+// `Permissions-Policy: camera=()`, which an iframe cannot escape). So for
+// screen+camera we capture the camera HERE in the offscreen document (extension
+// origin — always allowed) and draw it into a canvas on top of the screen, then
+// record the canvas. The face ends up in the video on every page.
+
+async function readyVideo(stream: MediaStream): Promise<HTMLVideoElement> {
+  const video = document.createElement("video");
+  video.muted = true;
+  video.playsInline = true;
+  video.srcObject = stream;
+  await waitForMetadata(video).catch(() => undefined);
+  await video.play().catch(() => undefined);
+  return video;
+}
+
+function drawCameraBubble(
+  ctx: CanvasRenderingContext2D,
+  camera: HTMLVideoElement,
+  canvas: HTMLCanvasElement,
+): void {
+  const size = Math.round(Math.min(canvas.width, canvas.height) * 0.22);
+  const margin = Math.round(Math.min(canvas.width, canvas.height) * 0.035);
+  const cx = margin + size / 2;
+  const cy = canvas.height - margin - size / 2;
+  const r = size / 2;
+  const vw = camera.videoWidth || size;
+  const vh = camera.videoHeight || size;
+  const scale = Math.max(size / vw, size / vh);
+  const dw = vw * scale;
+  const dh = vh * scale;
+
+  ctx.save();
+  ctx.beginPath();
+  ctx.arc(cx, cy, r, 0, Math.PI * 2);
+  ctx.closePath();
+  ctx.clip();
+  // Mirror the camera horizontally to match how people expect to see themselves.
+  ctx.translate(cx, cy);
+  ctx.scale(-1, 1);
+  ctx.drawImage(camera, -dw / 2, -dh / 2, dw, dh);
+  ctx.restore();
+
+  ctx.save();
+  ctx.beginPath();
+  ctx.arc(cx, cy, r - 1, 0, Math.PI * 2);
+  ctx.lineWidth = Math.max(3, Math.round(size * 0.03));
+  ctx.strokeStyle = "rgba(255,255,255,0.92)";
+  ctx.stroke();
+  ctx.restore();
+}
+
+type Compositor = {
+  videoTrack: MediaStreamTrack;
+  canvasStream: MediaStream;
+  stop: () => void;
+};
+
+async function buildCompositor(
+  displayStream: MediaStream,
+  cameraStream: MediaStream,
+  dims: { width: number; height: number },
+): Promise<Compositor> {
+  const displayVideo = await readyVideo(displayStream);
+  const cameraVideo = await readyVideo(cameraStream);
+  const canvas = document.createElement("canvas");
+  canvas.width = dims.width;
+  canvas.height = dims.height;
+  const ctx = canvas.getContext("2d");
+  if (!ctx) throw new Error("Canvas 2D context unavailable for compositing.");
+
+  let running = true;
+  let raf = 0;
+  const draw = () => {
+    if (!running) return;
+    ctx.drawImage(displayVideo, 0, 0, canvas.width, canvas.height);
+    if (cameraVideo.readyState >= 2) drawCameraBubble(ctx, cameraVideo, canvas);
+    raf = requestAnimationFrame(draw);
+  };
+  raf = requestAnimationFrame(draw);
+
+  const canvasStream = canvas.captureStream(30);
+  return {
+    videoTrack: canvasStream.getVideoTracks()[0],
+    canvasStream,
+    stop: () => {
+      running = false;
+      cancelAnimationFrame(raf);
+      displayVideo.srcObject = null;
+      cameraVideo.srcObject = null;
+    },
+  };
+}
+
 async function createMixedAudio(
   streams: MediaStream[],
 ): Promise<{ audioContext: AudioContext | null; tracks: MediaStreamTrack[] }> {
@@ -287,6 +415,13 @@ async function uploadChunk(
   const text = await res.text().catch(() => "");
   const data = text ? (JSON.parse(text) as UploadResult) : {};
   if (!res.ok) {
+    console.error(
+      "[clips-offscreen] chunk upload failed",
+      res.status,
+      "hadAuth:",
+      Boolean(recording.authToken),
+      text.slice(0, 200),
+    );
     throw new Error(
       data?.error || `Upload failed (${res.status}): ${text || res.statusText}`,
     );
@@ -310,6 +445,7 @@ function disposePrepared(): void {
 }
 
 function cleanup(recording: ActiveRecording): void {
+  recording.stopCompositor?.();
   stopStreams([recording.outputStream, ...recording.sourceStreams]);
   void recording.audioContext?.close().catch(() => undefined);
 }
@@ -329,22 +465,29 @@ async function acquire(message: AcquireMessage): Promise<{
   let displayStream: MediaStream | null = null;
   let micStream: MediaStream | null = null;
   let cameraStream: MediaStream | null = null;
+  const devices = await readDeviceIds();
 
   if (message.mode === "camera") {
     cameraStream = await navigator.mediaDevices.getUserMedia({
-      video: {
-        width: { ideal: 1280 },
-        height: { ideal: 720 },
-        facingMode: "user",
-      },
-      audio: message.includeMicrophone,
+      video: cameraConstraints(devices.video),
+      audio: message.includeMicrophone
+        ? devices.audio
+          ? { deviceId: { exact: devices.audio } }
+          : true
+        : false,
     });
   } else {
     // Native "Choose what to share" picker. This is the screenshot Steve showed.
     displayStream = await navigator.mediaDevices.getDisplayMedia(
       displayConstraints(message.surface),
     );
-    if (message.includeMicrophone) micStream = await getMicStream();
+    if (message.includeMicrophone)
+      micStream = await getMicStream(devices.audio);
+    // The screen+camera face comes from the on-page bubble (captured in the
+    // display pixels), NOT composited here: canvas/requestAnimationFrame does
+    // not run in a hidden offscreen document, so compositing produced an empty
+    // recording ("No chunks found"). We record the display stream directly.
+    void message.includeCamera;
   }
 
   const videoStream = displayStream ?? cameraStream;
@@ -401,8 +544,24 @@ async function begin(message: BeginMessage): Promise<{
   if (activeRecording) throw new Error("Clips is already recording.");
 
   const videoStream = ready.displayStream ?? ready.cameraStream;
-  const videoTrack = videoStream?.getVideoTracks()[0];
-  if (!videoTrack) throw new Error("Capture video track was lost.");
+  const directVideoTrack = videoStream?.getVideoTracks()[0];
+  if (!directVideoTrack) throw new Error("Capture video track was lost.");
+
+  // Screen + camera → composite the camera bubble into a canvas and record that,
+  // so the face is in the video even on pages that block the on-page bubble.
+  let compositor: Compositor | null = null;
+  let videoTrack = directVideoTrack;
+  if (ready.displayStream && ready.cameraStream) {
+    compositor = await buildCompositor(
+      ready.displayStream,
+      ready.cameraStream,
+      {
+        width: ready.width,
+        height: ready.height,
+      },
+    );
+    videoTrack = compositor.videoTrack;
+  }
 
   const audioInputs = [
     ...(ready.displayStream ? [ready.displayStream] : []),
@@ -417,7 +576,9 @@ async function begin(message: BeginMessage): Promise<{
   const mimeType = chooseMimeType();
   const recorder = new MediaRecorder(outputStream, {
     mimeType,
-    videoBitsPerSecond: 4_000_000,
+    // Crisp 1080p capture — matches the web/desktop recorders. Files upload
+    // directly (no client-side shrink), so we favor sharpness over a budget.
+    videoBitsPerSecond: 8_000_000,
     audioBitsPerSecond: 128_000,
   });
 
@@ -442,6 +603,7 @@ async function begin(message: BeginMessage): Promise<{
       ...(ready.displayStream ? [ready.displayStream] : []),
       ...(ready.micStream ? [ready.micStream] : []),
       ...(ready.cameraStream ? [ready.cameraStream] : []),
+      ...(compositor ? [compositor.canvasStream] : []),
     ],
     audioContext: mixedAudio.audioContext,
     chunkIndex: 0,
@@ -450,6 +612,10 @@ async function begin(message: BeginMessage): Promise<{
     cancelled: false,
     restarting: false,
     startTimer: null,
+    stopCompositor: compositor?.stop ?? null,
+    displaySource: ready.displayStream,
+    cameraSource: ready.cameraStream,
+    micSource: ready.micStream,
     dimensions: { width: ready.width, height: ready.height },
     hasAudio: outputStream.getAudioTracks().length > 0,
     hasCamera:
@@ -516,6 +682,7 @@ function startRecorderNow(recording: ActiveRecording): void {
   try {
     recording.recorder.start(2000);
     recording.startedAtMs = Date.now();
+    console.log("[clips-offscreen] recorder.start ok");
     reportStatus(recording.sessionId, "recording", {
       recordingId: recording.recordingId,
       width: recording.dimensions.width,
@@ -683,22 +850,18 @@ async function restart(
   recording.restarting = true;
   recording.cancelled = true;
   if (recording.recorder.state !== "inactive") recording.recorder.stop();
-  // Close only the old mixing context; keep the capture tracks alive.
+  // Tear down the old compositor + mixing context; keep the capture tracks alive
+  // so the next BEGIN can build a fresh compositor/recorder on the same streams.
+  recording.stopCompositor?.();
   void recording.audioContext?.close().catch(() => undefined);
   activeRecording = null;
 
-  const videoStream =
-    recording.sourceStreams.find((s) => s.getVideoTracks().length) ?? null;
-  const isCamera = recording.mode === "camera";
   prepared = {
     sessionId: recording.sessionId,
     mode: recording.mode,
-    displayStream: isCamera ? null : videoStream,
-    cameraStream: isCamera ? videoStream : null,
-    micStream:
-      recording.sourceStreams.find(
-        (s) => s.getAudioTracks().length && s !== videoStream,
-      ) ?? null,
+    displayStream: recording.displaySource,
+    cameraStream: recording.cameraSource,
+    micStream: recording.micSource,
     width: recording.dimensions.width,
     height: recording.dimensions.height,
     endedListener: null,