@agent-native/core 0.70.3 → 0.71.0

package/corpus/templates/clips/chrome-extension/src/background.ts

@@ -210,7 +210,11 @@ type OverlayPart = "bubble" | "countdown" | "toolbar" | "saving";
 let overlayPhase: OverlayPhase = "idle";
 let overlayBaseElapsedMs = 0;
 let overlayBaseEpochMs = 0;
+// Bubble during the countdown preview (any camera). During recording, the face
+// is composited into the video by the offscreen for screen+camera, so we only
+// show the on-page bubble for camera-only mode (recordingShowsBubble).
 let overlayShowsBubble = false;
+let recordingShowsBubble = false;
 let countdownEndsAtMs = 0;
 
 function desiredParts(): OverlayPart[] {
@@ -221,7 +225,7 @@ function desiredParts(): OverlayPart[] {
     return overlayShowsBubble ? ["bubble", "countdown"] : ["countdown"];
   }
   if (overlayPhase === "recording" || overlayPhase === "paused") {
-    return overlayShowsBubble ? ["bubble", "toolbar"] : ["toolbar"];
+    return recordingShowsBubble ? ["bubble", "toolbar"] : ["toolbar"];
   }
   if (overlayPhase === "saving") {
     return ["saving"];
@@ -286,6 +290,7 @@ function persistOverlay(): Promise<void> {
       baseElapsedMs: overlayBaseElapsedMs,
       baseEpochMs: overlayBaseEpochMs,
       showsBubble: overlayShowsBubble,
+      recordingShowsBubble,
       countdownEndsAtMs,
     },
   }).catch(() => undefined);
@@ -306,6 +311,7 @@ async function restoreRuntimeState(): Promise<void> {
         baseElapsedMs?: number;
         baseEpochMs?: number;
         showsBubble?: boolean;
+        recordingShowsBubble?: boolean;
         countdownEndsAtMs?: number;
       }
     | undefined;
@@ -316,6 +322,7 @@ async function restoreRuntimeState(): Promise<void> {
     overlayBaseEpochMs =
       typeof rt.baseEpochMs === "number" ? rt.baseEpochMs : 0;
     overlayShowsBubble = Boolean(rt.showsBubble);
+    recordingShowsBubble = Boolean(rt.recordingShowsBubble);
     countdownEndsAtMs =
       typeof rt.countdownEndsAtMs === "number" ? rt.countdownEndsAtMs : 0;
   }
@@ -413,6 +420,7 @@ function resetOverlay(): void {
   overlayBaseElapsedMs = 0;
   overlayBaseEpochMs = 0;
   overlayShowsBubble = false;
+  recordingShowsBubble = false;
   clearCountdownTimer();
   setRecordingFlag(false);
   // Bring back the toolbar popup now that we're idle again.
@@ -420,6 +428,41 @@ function resetOverlay(): void {
   void persistOverlay();
 }
 
+// ----- Pre-record camera preview --------------------------------------------
+// While the popup is open and idle, show the live face bubble on the active tab
+// (like the desktop app's pre-record bubble). The popup holds a runtime port
+// open; its disconnect (popup closed) tears the preview down.
+let previewTabId: number | null = null;
+
+async function startPreview(wantsCamera: boolean): Promise<void> {
+  await ensureRestored();
+  // Never show a preview over an active recording (the recording overlay owns
+  // the tab then). If the camera is off, make sure any preview is removed.
+  if (overlayPhase !== "idle" || !wantsCamera) {
+    await stopPreview();
+    return;
+  }
+  const tab = await queryActiveTab();
+  if (!tab || typeof tab.id !== "number") return;
+  if (previewTabId !== null && previewTabId !== tab.id) await stopPreview();
+  previewTabId = tab.id;
+  await ensureContentScript(tab.id);
+  // Injection / send fail silently on unsupported pages (chrome://, etc.).
+  await sendTabMessage(tab.id, {
+    type: "CLIPS_OVERLAY_MOUNT",
+    parts: ["bubble"],
+  });
+}
+
+async function stopPreview(): Promise<void> {
+  const tabId = previewTabId;
+  previewTabId = null;
+  if (tabId === null) return;
+  // Don't tear down a recording overlay that took over this tab.
+  if (overlayPhase !== "idle") return;
+  await sendTabMessage(tabId, { type: "CLIPS_OVERLAY_UNMOUNT" });
+}
+
 // Mirrored into chrome.storage.local so content scripts can cheaply tell whether
 // a recording is in progress without waking the service worker on every page
 // load. Content scripts can read storage.local by default; storage.session
@@ -875,6 +918,7 @@ async function armRecording(args: {
     settings.captureSurface === "camera" ? "camera" : "screen";
   const surface: "browser" | "window" | "monitor" =
     settings.captureSurface === "camera" ? "browser" : settings.captureSurface;
+  console.log("[clips-bg] arm: start", { mode, surface, tabId: tab.id });
 
   // Pre-warmed on popup open, so this is effectively instant and keeps the
   // getDisplayMedia() call close to the user's click.
@@ -882,15 +926,51 @@ async function armRecording(args: {
 
   // 1) Native "Choose what to share" picker. Do this before any network round
   //    trip so the picker stays tied to the user gesture. Throws if cancelled.
-  await sendOffscreenMessage<{ ok: boolean; width: number; height: number }>({
+  const acq = await sendOffscreenMessage<{
+    ok: boolean;
+    width: number;
+    height: number;
+  }>({
     type: "CLIPS_OFFSCREEN_ACQUIRE",
     sessionId,
     mode,
     surface,
     includeMicrophone: settings.includeMicrophone,
+    includeCamera: settings.includeCamera,
   });
+  console.log("[clips-bg] arm: acquired stream", acq);
+
+  // 2) Show the countdown overlay IMMEDIATELY — before the network round-trip —
+  //    so the 3-2-1 runs full-length and the dim/blur clears exactly when the
+  //    recorder starts. The on-page bubble shows during countdown AND recording
+  //    (the face is captured in the display; we do not composite).
+  const cameraInvolved = mode === "camera" || settings.includeCamera;
+  overlayPhase = "countdown";
+  overlayBaseElapsedMs = 0;
+  overlayBaseEpochMs = nowMs();
+  overlayShowsBubble = cameraInvolved;
+  recordingShowsBubble = cameraInvolved;
+  countdownEndsAtMs = nowMs() + COUNTDOWN_SECONDS * 1000;
+  // The recording overlay now owns this tab's bubble — hand off from any preview
+  // so the popup-close disconnect won't tear down the live recording overlay.
+  previewTabId = null;
+  setRecordingFlag(true);
+  // Clicking the icon now stops & saves immediately instead of opening the popup.
+  setActionPopup("");
+  await mountOverlayOnTab(tab.id as number);
+  broadcastOverlayState();
+
+  const abortArming = async (): Promise<void> => {
+    await sendOffscreenMessage({
+      type: "CLIPS_OFFSCREEN_CANCEL",
+      sessionId,
+    }).catch(() => undefined);
+    resetOverlay();
+    await broadcastUnmount();
+    await chrome.action.setBadgeText({ text: "" });
+  };
 
-  // 2) Create the recording row now that we know the user picked a source.
+  // 3) Create the recording row (happens during the countdown).
   type CreatedRecording = {
     id?: string;
     uploadChunkUrl?: string;
@@ -903,23 +983,17 @@ async function armRecording(args: {
       titleSource: tab.title ? "context" : "default",
       sourceAppName: "Chrome",
       sourceWindowTitle: tab.title ?? null,
-      hasCamera: settings.captureSurface === "camera" || settings.includeCamera,
+      hasCamera: cameraInvolved,
       hasAudio:
         settings.includeMicrophone || settings.captureSurface !== "camera",
       visibility: "public",
     });
   } catch (err) {
-    await sendOffscreenMessage({
-      type: "CLIPS_OFFSCREEN_CANCEL",
-      sessionId,
-    }).catch(() => undefined);
+    await abortArming();
     throw err;
   }
   if (!created.id || !created.uploadChunkUrl) {
-    await sendOffscreenMessage({
-      type: "CLIPS_OFFSCREEN_CANCEL",
-      sessionId,
-    }).catch(() => undefined);
+    await abortArming();
     throw new Error("Clips did not create a recording target.");
   }
 
@@ -946,35 +1020,24 @@ async function armRecording(args: {
     error: null,
   };
 
-  // 3) Enter the countdown phase and show the overlay on the launch tab. The
-  //    camera bubble shows whenever a camera is involved: as the picture-in-
-  //    picture for screen+camera, or as a live self-preview for camera-only.
-  overlayPhase = "countdown";
-  overlayBaseElapsedMs = 0;
-  overlayBaseEpochMs = nowMs();
-  overlayShowsBubble = mode === "camera" || settings.includeCamera;
-  countdownEndsAtMs = nowMs() + COUNTDOWN_SECONDS * 1000;
-  setRecordingFlag(true);
-  // Clicking the icon now stops & saves immediately instead of opening the popup.
-  setActionPopup("");
-
-  // Hand the recorder its targets plus the pre-roll delay. The offscreen document
-  // (a reliable context, unlike the suspendable worker) owns the countdown timer
-  // and reports "recording" back when the recorder actually starts — that status
-  // flip (markRecordingStarted) advances our phase, so recording begins even on
-  // pages where no overlay can be injected (chrome://, the New Tab page, etc.).
+  // 4) Start the recorder when the (already-running) countdown ends. The
+  //    offscreen owns the pre-roll timer (a reliable context, unlike the
+  //    suspendable worker) and reports "recording" back when it actually starts.
   const authToken = (await readAuthSession(settings))?.token;
+  console.log("[clips-bg] arm: created row", created.id, "auth?", !!authToken);
+  const startDelayMs = Math.max(0, countdownEndsAtMs - nowMs());
   try {
     await sendOffscreenMessage({
       type: "CLIPS_OFFSCREEN_BEGIN",
       sessionId,
       recordingId: created.id,
       uploadUrl,
-      hasCamera: mode === "camera" || settings.includeCamera,
-      startDelayMs: COUNTDOWN_SECONDS * 1000,
+      hasCamera: cameraInvolved,
+      startDelayMs,
       authToken,
     });
   } catch (err) {
+    console.error("[clips-bg] arm: BEGIN failed", err);
     await cancelRecording();
     throw err;
   }
@@ -985,10 +1048,10 @@ async function armRecording(args: {
     await attachSession(session);
   }
 
-  await mountOverlayOnTab(tab.id as number);
   broadcastOverlayState();
   await chrome.action.setBadgeBackgroundColor({ color: "#e11d48" });
   await chrome.action.setBadgeText({ text: "REC" });
+  console.log("[clips-bg] arm: done — countdown, parts:", desiredParts());
 
   return { ok: true, sessionId, recordingId: created.id, native: true };
 }
@@ -998,6 +1061,7 @@ async function armRecording(args: {
 // on-page timer. Idempotent.
 async function markRecordingStarted() {
   if (overlayPhase !== "countdown") return;
+  console.log("[clips-bg] recorder started → phase=recording");
   clearCountdownTimer();
   overlayPhase = "recording";
   overlayBaseElapsedMs = 0;
@@ -1178,6 +1242,7 @@ async function finishSaving(
 
 async function stopRecording() {
   const recording = activeNativeRecording;
+  console.log("[clips-bg] stopRecording — active:", !!recording);
   if (!recording) return { ok: false, error: "No active Clips recording." };
   recording.status = "stopping";
   // Keep an overlay up: swap the recording controls/bubble for a "Saving…" card
@@ -1821,6 +1886,11 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
     try {
       response = await dispatchRuntimeMessage(message);
     } catch (err) {
+      console.error(
+        "[clips-bg] message failed:",
+        (message as { type?: unknown })?.type,
+        err,
+      );
       response = {
         ok: false,
         error: err instanceof Error ? err.message : "Could not use Clips.",
@@ -1841,6 +1911,13 @@ async function dispatchRuntimeMessage(message: unknown): Promise<unknown> {
   // Status updates streamed from the offscreen recorder.
   if (type === "CLIPS_NATIVE_STATUS") {
     const status = message as OffscreenStatusMessage;
+    console.log(
+      "[clips-bg] status:",
+      status.status,
+      status.error ?? "",
+      "phase:",
+      overlayPhase,
+    );
     if (
       activeNativeRecording &&
       activeNativeRecording.sessionId === status.sessionId
@@ -1909,6 +1986,11 @@ async function dispatchRuntimeMessage(message: unknown): Promise<unknown> {
     return { ok: true };
   }
 
+  if (type === "CLIPS_PREVIEW") {
+    await startPreview(Boolean((message as { camera?: unknown }).camera));
+    return { ok: true };
+  }
+
   switch (type) {
     case "CLIPS_POPUP_START":
       return handlePopupStart(message as PopupStartMessage);
@@ -1961,6 +2043,12 @@ chrome.action.onClicked.addListener(() => {
     // Await restore — the click may have woken the worker, leaving the module
     // globals empty until this resolves. Without it, Stop silently does nothing.
     await ensureRestored();
+    console.log(
+      "[clips-bg] icon clicked — phase:",
+      overlayPhase,
+      "active:",
+      !!activeNativeRecording,
+    );
     if (!activeNativeRecording) return;
     if (overlayPhase === "countdown") await cancelRecording();
     else if (overlayPhase === "recording" || overlayPhase === "paused") {
@@ -1969,6 +2057,16 @@ chrome.action.onClicked.addListener(() => {
   })();
 });
 
+// The popup opens a port for as long as it is visible; its disconnect (popup
+// closed) is our reliable signal to remove the pre-record preview bubble.
+chrome.runtime.onConnect.addListener((port) => {
+  if (port.name !== "clips-preview") return;
+  port.onDisconnect.addListener(() => {
+    void chrome.runtime.lastError;
+    void stopPreview();
+  });
+});
+
 chrome.runtime.onMessageExternal.addListener(
   (message, sender, sendResponse) => {
     void handleExternalMessage(message as ExternalMessage, sender)

package/corpus/templates/clips/chrome-extension/src/content-script.ts

@@ -13,6 +13,128 @@
   const ALL_PARTS: OverlayPart[] = ["bubble", "countdown", "toolbar", "saving"];
   const flags = window as unknown as { __clipsOverlayHostReady?: boolean };
 
+  // ----- Draggable, resizable camera bubble ---------------------------------
+  // Size + position persist in storage so the bubble stays where the user put it
+  // across pages and recordings (like the desktop app). The iframe can't move or
+  // resize itself, so the content script owns its geometry.
+  const BUBBLE_SIZES: Record<string, number> = { sm: 184, lg: 280 };
+  const bubbleGeom: { size: string; left: number | null; top: number | null } =
+    { size: "lg", left: null, top: null };
+  let bubbleDragLayer: HTMLDivElement | null = null;
+  let bubblePersistTimer: ReturnType<typeof setTimeout> | undefined;
+
+  function bubbleSizePx(): number {
+    return BUBBLE_SIZES[bubbleGeom.size] ?? BUBBLE_SIZES.lg;
+  }
+
+  function clampBubble(
+    left: number,
+    top: number,
+    size: number,
+  ): { left: number; top: number } {
+    return {
+      left: Math.max(8, Math.min(left, window.innerWidth - size - 8)),
+      top: Math.max(8, Math.min(top, window.innerHeight - size - 8)),
+    };
+  }
+
+  function applyBubbleGeom(): void {
+    const frame = document.getElementById(
+      partFrameId("bubble"),
+    ) as HTMLIFrameElement | null;
+    if (!frame) return;
+    const size = bubbleSizePx();
+    const margin = 24;
+    const base = clampBubble(
+      bubbleGeom.left ?? margin,
+      bubbleGeom.top ?? window.innerHeight - size - margin,
+      size,
+    );
+    Object.assign(frame.style, {
+      left: `${base.left}px`,
+      top: `${base.top}px`,
+      bottom: "auto",
+      width: `${size}px`,
+      height: `${size}px`,
+    });
+  }
+
+  function persistBubbleGeom(): void {
+    clearTimeout(bubblePersistTimer);
+    bubblePersistTimer = setTimeout(() => {
+      try {
+        chrome.storage.local.set({ bubbleGeom });
+      } catch {
+        /* ignore */
+      }
+    }, 200);
+  }
+
+  function startBubbleDrag(): void {
+    if (bubbleDragLayer) return;
+    const frame = document.getElementById(
+      partFrameId("bubble"),
+    ) as HTMLIFrameElement | null;
+    if (!frame) return;
+    // Full-screen capture layer so the pointer keeps tracking after it leaves
+    // the small bubble iframe.
+    const layer = document.createElement("div");
+    Object.assign(layer.style, {
+      position: "fixed",
+      inset: "0",
+      zIndex: "2147483647",
+      cursor: "grabbing",
+    });
+    (document.documentElement || document.body).appendChild(layer);
+    bubbleDragLayer = layer;
+    const onMove = (e: PointerEvent): void => {
+      const size = bubbleSizePx();
+      const rect = frame.getBoundingClientRect();
+      const next = clampBubble(
+        rect.left + e.movementX,
+        rect.top + e.movementY,
+        size,
+      );
+      bubbleGeom.left = next.left;
+      bubbleGeom.top = next.top;
+      Object.assign(frame.style, {
+        left: `${next.left}px`,
+        top: `${next.top}px`,
+        bottom: "auto",
+      });
+    };
+    const onUp = (): void => {
+      layer.removeEventListener("pointermove", onMove);
+      layer.removeEventListener("pointerup", onUp);
+      layer.removeEventListener("pointercancel", onUp);
+      layer.remove();
+      bubbleDragLayer = null;
+      persistBubbleGeom();
+    };
+    layer.addEventListener("pointermove", onMove);
+    layer.addEventListener("pointerup", onUp);
+    layer.addEventListener("pointercancel", onUp);
+  }
+
+  try {
+    chrome.storage.local.get("bubbleGeom", (value) => {
+      if (chrome.runtime.lastError) return;
+      const g = value.bubbleGeom as
+        | { size?: unknown; left?: unknown; top?: unknown }
+        | undefined;
+      if (g && typeof g === "object") {
+        bubbleGeom.size = g.size === "sm" ? "sm" : "lg";
+        bubbleGeom.left = typeof g.left === "number" ? g.left : null;
+        bubbleGeom.top = typeof g.top === "number" ? g.top : null;
+      }
+      applyBubbleGeom();
+    });
+  } catch {
+    /* ignore */
+  }
+
+  window.addEventListener("resize", () => applyBubbleGeom());
+
   function requestState(): void {
     try {
       chrome.runtime.sendMessage(
@@ -78,21 +200,25 @@
     frame.setAttribute("allowtransparency", "true");
     if (part === "bubble") {
       frame.allow = "camera; microphone";
+      // Above the countdown so the face stays sharp over the dim/blur. Exact
+      // size/position are set by applyBubbleGeom() once mounted.
+      const size = bubbleSizePx();
       Object.assign(frame.style, {
         left: "24px",
         bottom: "24px",
-        width: "148px",
-        height: "148px",
+        width: `${size}px`,
+        height: `${size}px`,
+        zIndex: "3",
       });
     } else if (part === "toolbar") {
       // Left-edge vertical pill (desktop layout). Height grows on hover via the
       // resize message below.
       Object.assign(frame.style, {
         left: "16px",
-        top: "50%",
-        transform: "translateY(-50%)",
+        top: "calc(50% - 77px)",
         width: "68px",
         height: "154px",
+        zIndex: "2",
       });
     } else if (part === "saving") {
       Object.assign(frame.style, {
@@ -100,12 +226,15 @@
         bottom: "24px",
         width: "264px",
         height: "96px",
+        zIndex: "2",
       });
     } else {
+      // countdown — full-screen dim/blur, below the bubble.
       Object.assign(frame.style, {
         inset: "0",
         width: "100%",
         height: "100%",
+        zIndex: "1",
       });
     }
   }
@@ -114,15 +243,18 @@
     if (document.getElementById(partFrameId(part))) return;
     const frame = document.createElement("iframe");
     frame.id = partFrameId(part);
+    if (part === "bubble") frame.allow = "camera; microphone";
     const url = new URL(chrome.runtime.getURL("src/overlay.html"));
     url.searchParams.set("part", part);
     if (part === "countdown") url.searchParams.set("seconds", "3");
     frame.src = url.toString();
     styleFrame(frame, part);
     container.appendChild(frame);
+    if (part === "bubble") applyBubbleGeom();
   }
 
   function reconcile(parts: OverlayPart[]): void {
+    console.log("[clips-cs] reconcile parts:", parts, "on", location.href);
     const wanted = new Set(parts.filter((p) => ALL_PARTS.includes(p)));
     if (wanted.size === 0) {
       document.getElementById(CONTAINER_ID)?.remove();
@@ -157,20 +289,38 @@
     }
   });
 
-  // The toolbar overlay asks to grow/shrink its own iframe on hover (it can't
-  // resize itself). Only trust messages from our own extension-origin frames.
+  // Overlay iframes post layout requests (toolbar hover-resize, bubble drag and
+  // size). Only trust messages from our own extension-origin frames.
   window.addEventListener("message", (event) => {
     const data = event.data as
-      | { source?: string; kind?: string; part?: string; height?: number }
+      | {
+          source?: string;
+          kind?: string;
+          part?: string;
+          height?: number;
+          size?: string;
+        }
       | undefined;
-    if (!data || data.source !== "clips-overlay" || data.kind !== "resize") {
+    if (!data || data.source !== "clips-overlay") return;
+    if (event.origin !== chrome.runtime.getURL("").replace(/\/$/, "")) return;
+
+    if (data.kind === "resize" && data.part === "toolbar") {
+      const frame = document.getElementById(partFrameId("toolbar"));
+      if (frame && typeof data.height === "number") {
+        frame.style.height = `${Math.round(data.height)}px`;
+      }
+      return;
+    }
+    if (data.kind === "bubble-drag-start") {
+      startBubbleDrag();
+      return;
+    }
+    if (data.kind === "bubble-size") {
+      bubbleGeom.size = data.size === "sm" ? "sm" : "lg";
+      applyBubbleGeom();
+      persistBubbleGeom();
       return;
     }
-    if (event.origin !== chrome.runtime.getURL("").replace(/\/$/, "")) return;
-    const part = data.part === "toolbar" ? "toolbar" : null;
-    if (!part || typeof data.height !== "number") return;
-    const frame = document.getElementById(partFrameId(part));
-    if (frame) frame.style.height = `${Math.round(data.height)}px`;
   });
 
   syncIfRecording();