@agent-native/core 0.87.0 → 0.90.1

package/corpus/templates/clips/chrome-extension/src/offscreen.ts

@@ -16,6 +16,12 @@
 // MIME selection and the chunk-upload URL/param protocol are shared with the web
 // app recorder via @shared/recording-core so the server contract can't drift.
 
+import {
+  chooseFallbackAudioInput,
+  enumerateAudioInputDevices,
+  isLikelyPhoneMicLabel,
+  type AudioInputFallback,
+} from "@shared/media-device-selection";
 import { scheduleReadyChime } from "@shared/recording-audio";
 import {
   chunkUploadUrl,
@@ -78,6 +84,8 @@ type AcquireMessage = {
   // Screen+camera: capture the camera here and composite it into the recording
   // (the on-page bubble can be blocked by the page's Permissions-Policy).
   includeCamera: boolean;
+  videoDeviceId?: string;
+  audioDeviceId?: string;
 };
 
 type BeginMessage = {
@@ -281,15 +289,50 @@ function displayConstraints(
 }
 
 // The user's chosen camera/mic devices (set in the popup, saved to storage).
-async function readDeviceIds(): Promise<{ video: string; audio: string }> {
+async function readDeviceIds(overrides?: {
+  video?: string;
+  audio?: string;
+}): Promise<{ video: string; audio: string }> {
   try {
     const v = await chrome.storage.sync.get(["videoDeviceId", "audioDeviceId"]);
     return {
-      video: typeof v.videoDeviceId === "string" ? v.videoDeviceId : "",
-      audio: typeof v.audioDeviceId === "string" ? v.audioDeviceId : "",
+      video:
+        typeof overrides?.video === "string"
+          ? overrides.video
+          : typeof v.videoDeviceId === "string"
+            ? v.videoDeviceId
+            : "",
+      audio:
+        typeof overrides?.audio === "string"
+          ? overrides.audio
+          : typeof v.audioDeviceId === "string"
+            ? v.audioDeviceId
+            : "",
     };
   } catch {
-    return { video: "", audio: "" };
+    return {
+      video: typeof overrides?.video === "string" ? overrides.video : "",
+      audio: typeof overrides?.audio === "string" ? overrides.audio : "",
+    };
+  }
+}
+
+// Best-effort label lookup for the requested mic id, used only to soften audio
+// processing constraints for phone/Continuity-style mics (see
+// isLikelyPhoneMic). Device labels are only populated once permission has
+// already been granted, which is always true by the time acquire() runs.
+async function lookupAudioDeviceLabel(deviceId: string): Promise<string> {
+  if (!deviceId) return "";
+  try {
+    const devices = await navigator.mediaDevices.enumerateDevices();
+    return (
+      devices.find(
+        (device) =>
+          device.kind === "audioinput" && device.deviceId === deviceId,
+      )?.label ?? ""
+    );
+  } catch {
+    return "";
   }
 }
 
@@ -303,21 +346,294 @@ function cameraConstraints(deviceId: string): MediaTrackConstraints {
   return base;
 }
 
-async function getMicStream(deviceId: string): Promise<MediaStream | null> {
+async function getCameraStream(deviceId: string): Promise<MediaStream> {
+  try {
+    const stream = await navigator.mediaDevices.getUserMedia({
+      video: cameraConstraints(deviceId),
+      audio: false,
+    });
+    warnIfTrackDeviceMismatch("camera", deviceId, stream.getVideoTracks()[0]);
+    return stream;
+  } catch (error) {
+    if (!deviceId || !isDeviceUnavailableError(error)) throw error;
+
+    captureExtensionError(
+      new Error("Selected Clips camera was unavailable; using default camera."),
+      {
+        tags: { surface: "offscreen", mechanism: "camera-device-fallback" },
+        extra: {
+          requestedDeviceId: deviceId,
+          originalError:
+            error instanceof Error
+              ? { name: error.name, message: error.message }
+              : String(error),
+        },
+      },
+    );
+
+    return navigator.mediaDevices.getUserMedia({
+      video: cameraConstraints(""),
+      audio: false,
+    });
+  }
+}
+
+// Phones/Continuity-style mics already apply their own echo cancellation,
+// noise suppression, and gain control before the audio ever reaches Chrome.
+// Stacking Chrome's versions of the same processing on top double-processes
+// the signal and audibly degrades it. We can't detect "is this device doing
+// its own processing" directly, so we use the device label as a heuristic and
+// ask the browser to SKIP its own noise suppression / AGC for those devices
+// (`{ ideal: false }` — a best-effort request, not a hard requirement). Echo
+// cancellation stays on for everything.
+async function chooseFallbackMicDevice(
+  requestedLabel: string,
+  avoidDeviceIds: string[] = [],
+): Promise<AudioInputFallback | null> {
+  try {
+    return chooseFallbackAudioInput(await enumerateAudioInputDevices(), {
+      savedLabel: requestedLabel,
+      avoidDeviceIds,
+    });
+  } catch {
+    return null;
+  }
+}
+
+function voiceFocusedAudioConstraints(
+  deviceId: string,
+  deviceLabel = "",
+): MediaTrackConstraints {
+  const skipRedundantProcessing = isLikelyPhoneMicLabel(deviceLabel);
   const audio: MediaTrackConstraints = {
     echoCancellation: true,
-    noiseSuppression: true,
-    autoGainControl: true,
+    noiseSuppression: skipRedundantProcessing ? { ideal: false } : true,
+    autoGainControl: skipRedundantProcessing ? { ideal: false } : true,
     channelCount: 1,
   };
   if (deviceId) audio.deviceId = { exact: deviceId };
+  return audio;
+}
+
+function isDeviceUnavailableError(error: unknown): boolean {
+  const name =
+    error && typeof error === "object" && "name" in error
+      ? String((error as { name?: unknown }).name)
+      : "";
+  return (
+    name === "OverconstrainedError" ||
+    name === "NotFoundError" ||
+    name === "DevicesNotFoundError"
+  );
+}
+
+function micLabelDiagnostic(label: string | null | undefined): string {
+  const value = label?.trim();
+  if (!value) return "empty";
+  if (isLikelyPhoneMicLabel(value)) return "phone-like";
+  if (/\b(?:macbook|built[- ]?in|internal microphone)\b/i.test(value)) {
+    return "built-in";
+  }
+  return "redacted";
+}
+
+async function getMicStream(
+  deviceId: string,
+  deviceLabel = "",
+): Promise<MediaStream> {
+  const requestedDeviceId = deviceId;
+  const requestedDeviceLabel = deviceLabel;
+  const audio = voiceFocusedAudioConstraints(
+    requestedDeviceId,
+    requestedDeviceLabel,
+  );
   try {
-    return await navigator.mediaDevices.getUserMedia({ audio, video: false });
-  } catch {
-    return null;
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio,
+      video: false,
+    });
+    return await correctMicStreamIfNeeded(
+      stream,
+      requestedDeviceId,
+      requestedDeviceLabel,
+    );
+  } catch (error) {
+    if (!requestedDeviceId || !isDeviceUnavailableError(error)) throw error;
+
+    captureExtensionError(
+      new Error(
+        "Selected Clips microphone was unavailable; retrying an explicit fallback mic.",
+      ),
+      {
+        tags: { surface: "offscreen", mechanism: "mic-device-fallback" },
+        extra: {
+          requestedDeviceId,
+          requestedDeviceLabel: micLabelDiagnostic(requestedDeviceLabel),
+          originalError:
+            error instanceof Error
+              ? { name: error.name, message: error.message }
+              : String(error),
+        },
+      },
+    );
+
+    const fallback = await chooseFallbackMicDevice(requestedDeviceLabel, [
+      requestedDeviceId,
+    ]);
+    if (fallback) {
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          audio: voiceFocusedAudioConstraints(
+            fallback.deviceId,
+            fallback.label,
+          ),
+          video: false,
+        });
+        return correctMicStreamIfNeeded(
+          stream,
+          fallback.deviceId,
+          fallback.label,
+        );
+      } catch (fallbackError) {
+        if (!isDeviceUnavailableError(fallbackError)) throw fallbackError;
+        captureExtensionError(
+          new Error("Explicit Clips microphone fallback was unavailable."),
+          {
+            tags: { surface: "offscreen", mechanism: "mic-device-fallback" },
+            extra: {
+              requestedDeviceId,
+              requestedDeviceLabel: micLabelDiagnostic(requestedDeviceLabel),
+              fallbackDeviceId: fallback.deviceId,
+              fallbackDeviceLabel: micLabelDiagnostic(fallback.label),
+              fallbackReason: fallback.reason,
+              originalError:
+                fallbackError instanceof Error
+                  ? {
+                      name: fallbackError.name,
+                      message: fallbackError.message,
+                    }
+                  : String(fallbackError),
+            },
+          },
+        );
+      }
+    }
+
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio: voiceFocusedAudioConstraints("", ""),
+      video: false,
+    });
+    return correctMicStreamIfNeeded(stream, "", "");
   }
 }
 
+async function correctMicStreamIfNeeded(
+  stream: MediaStream,
+  requestedDeviceId: string,
+  requestedDeviceLabel: string,
+): Promise<MediaStream> {
+  const track = stream.getAudioTracks()[0];
+  if (!track) return stream;
+  const settings = track.getSettings?.();
+  const actualDeviceId = settings?.deviceId ?? "";
+  const mismatched =
+    !!requestedDeviceId &&
+    !!actualDeviceId &&
+    actualDeviceId !== requestedDeviceId;
+  const phoneLike = isLikelyPhoneMicLabel(track.label);
+  if (!mismatched && !phoneLike) return stream;
+
+  warnIfTrackDeviceMismatch("mic", requestedDeviceId, track);
+  if (phoneLike) {
+    captureExtensionError(
+      new Error(
+        "Captured microphone looked phone-like; retrying fallback mic.",
+      ),
+      {
+        tags: {
+          surface: "offscreen",
+          mechanism: "mic-phone-capture-correction",
+        },
+        extra: {
+          requestedDeviceId,
+          requestedDeviceLabel: micLabelDiagnostic(requestedDeviceLabel),
+          actualDeviceId,
+          trackLabel: micLabelDiagnostic(track.label),
+        },
+      },
+    );
+  }
+  const fallback = await chooseFallbackMicDevice(requestedDeviceLabel, [
+    requestedDeviceId,
+    actualDeviceId,
+  ]);
+  if (!fallback) return stream;
+
+  try {
+    const replacement = await navigator.mediaDevices.getUserMedia({
+      audio: voiceFocusedAudioConstraints(fallback.deviceId, fallback.label),
+      video: false,
+    });
+    stream.getTracks().forEach((oldTrack) => oldTrack.stop());
+    return replacement;
+  } catch (error) {
+    captureExtensionError(
+      new Error("Corrective microphone fallback failed; keeping original mic."),
+      {
+        tags: { surface: "offscreen", mechanism: "mic-correction-failed" },
+        extra: {
+          requestedDeviceId,
+          requestedDeviceLabel: micLabelDiagnostic(requestedDeviceLabel),
+          fallbackDeviceId: fallback.deviceId,
+          fallbackDeviceLabel: micLabelDiagnostic(fallback.label),
+          fallbackReason: fallback.reason,
+          originalError:
+            error instanceof Error
+              ? { name: error.name, message: error.message }
+              : String(error),
+        },
+      },
+    );
+    return stream;
+  }
+}
+
+// Defense in depth against the popup/storage layer ever saving a "default" or
+// stale device id again (see popup.ts device-menu filtering): after acquiring
+// a stream, compare what was actually granted against what was requested, and
+// surface a loud warning + Sentry breadcrumb if they don't match. Chrome
+// resolves `deviceId: { exact: "<id>" }` against the device active AT CAPTURE
+// TIME, so a mismatch here means the OS silently switched inputs (e.g. macOS
+// Continuity promoting a nearby iPhone to system default).
+function warnIfTrackDeviceMismatch(
+  kind: "mic" | "camera",
+  requestedDeviceId: string,
+  track: MediaStreamTrack | undefined,
+): void {
+  if (!requestedDeviceId || !track) return;
+  const settings = track.getSettings?.();
+  const actualDeviceId = settings?.deviceId ?? "";
+  if (actualDeviceId && actualDeviceId === requestedDeviceId) return;
+  console.warn(
+    `[clips-offscreen] ${kind} device mismatch: requested`,
+    requestedDeviceId,
+    "but captured",
+    actualDeviceId || "(unknown)",
+    `label="${track.label}"`,
+  );
+  captureExtensionError(
+    new Error(`Captured ${kind} device did not match the requested device.`),
+    {
+      tags: { surface: "offscreen", mechanism: `${kind}-device-mismatch` },
+      extra: {
+        requestedDeviceId,
+        actualDeviceId,
+        trackLabel: micLabelDiagnostic(track.label),
+      },
+    },
+  );
+}
+
 /* ----------------------------------------------------- camera compositing --- */
 
 // On many pages the on-page camera bubble can't run (the page sets
@@ -711,58 +1027,66 @@ async function acquire(message: AcquireMessage): Promise<{
   let displayStream: MediaStream | null = null;
   let micStream: MediaStream | null = null;
   let cameraStream: MediaStream | null = null;
-  const devices = await readDeviceIds();
-
-  if (message.mode === "camera") {
-    cameraStream = await navigator.mediaDevices.getUserMedia({
-      video: cameraConstraints(devices.video),
-      audio: message.includeMicrophone
-        ? devices.audio
-          ? { deviceId: { exact: devices.audio } }
-          : true
-        : false,
-    });
-  } else {
-    // Native "Choose what to share" picker. This is the screenshot Steve showed.
-    displayStream = await navigator.mediaDevices.getDisplayMedia(
-      displayConstraints(message.surface),
-    );
-    if (message.includeMicrophone)
-      micStream = await getMicStream(devices.audio);
-    // The screen+camera face comes from the on-page bubble (captured in the
-    // display pixels), NOT composited here: canvas/requestAnimationFrame does
-    // not run in a hidden offscreen document, so compositing produced an empty
-    // recording ("No chunks found"). We record the display stream directly.
-    void message.includeCamera;
-  }
+  const devices = await readDeviceIds({
+    video: message.videoDeviceId,
+    audio: message.audioDeviceId,
+  });
+
+  try {
+    if (message.mode === "camera") {
+      cameraStream = await getCameraStream(devices.video);
+      if (message.includeMicrophone) {
+        const audioLabel = await lookupAudioDeviceLabel(devices.audio);
+        micStream = await getMicStream(devices.audio, audioLabel);
+      }
+    } else {
+      // Native "Choose what to share" picker. This is the screenshot Steve showed.
+      displayStream = await navigator.mediaDevices.getDisplayMedia(
+        displayConstraints(message.surface),
+      );
+      if (message.includeMicrophone) {
+        const audioLabel = await lookupAudioDeviceLabel(devices.audio);
+        micStream = await getMicStream(devices.audio, audioLabel);
+      }
+      // The screen+camera face comes from the on-page bubble (captured in the
+      // display pixels), NOT composited here: canvas/requestAnimationFrame does
+      // not run in a hidden offscreen document, so compositing produced an empty
+      // recording ("No chunks found"). We record the display stream directly.
+      void message.includeCamera;
+    }
 
-  const videoStream = displayStream ?? cameraStream;
-  if (!videoStream) throw new Error("No media stream was available to record.");
-  const { width, height } = await streamDimensions(videoStream);
+    const videoStream = displayStream ?? cameraStream;
+    if (!videoStream)
+      throw new Error("No media stream was available to record.");
+    const { width, height } = await streamDimensions(videoStream);
+
+    // If the user stops sharing via Chrome's native control, tell the worker so it
+    // can run the normal stop/finalize flow.
+    const endedTrack = videoStream.getVideoTracks()[0] ?? null;
+    const endedListener = () => {
+      chrome.runtime.sendMessage({
+        type: "CLIPS_NATIVE_ENDED",
+        sessionId: message.sessionId,
+      });
+    };
+    endedTrack?.addEventListener("ended", endedListener);
 
-  // If the user stops sharing via Chrome's native control, tell the worker so it
-  // can run the normal stop/finalize flow.
-  const endedTrack = videoStream.getVideoTracks()[0] ?? null;
-  const endedListener = () => {
-    chrome.runtime.sendMessage({
-      type: "CLIPS_NATIVE_ENDED",
+    prepared = {
       sessionId: message.sessionId,
-    });
-  };
-  endedTrack?.addEventListener("ended", endedListener);
-
-  prepared = {
-    sessionId: message.sessionId,
-    mode: message.mode,
-    displayStream,
-    micStream,
-    cameraStream,
-    width,
-    height,
-    endedListener,
-    endedTrack,
-  };
-  return { ok: true, width, height };
+      mode: message.mode,
+      displayStream,
+      micStream,
+      cameraStream,
+      width,
+      height,
+      endedListener,
+      endedTrack,
+    };
+    return { ok: true, width, height };
+  } catch (error) {
+    stopStreams([displayStream, micStream, cameraStream]);
+    throw error;
+  }
 }
 
 function stopPreparedStreams(): void {

package/corpus/templates/clips/chrome-extension/src/overlay.ts

@@ -17,6 +17,56 @@ type OverlayPhase = "idle" | "countdown" | "recording" | "paused" | "saving";
 const TOOLBAR_COLLAPSED_H = 154;
 const TOOLBAR_EXPANDED_H = 236;
 
+function isDeviceUnavailableError(error: unknown): boolean {
+  const name =
+    error && typeof error === "object" && "name" in error
+      ? String((error as { name?: unknown }).name)
+      : "";
+  return (
+    name === "OverconstrainedError" ||
+    name === "NotFoundError" ||
+    name === "DevicesNotFoundError"
+  );
+}
+
+function cameraConstraint(deviceId: string): MediaTrackConstraints {
+  const video: MediaTrackConstraints = {
+    width: { ideal: 640 },
+    height: { ideal: 640 },
+  };
+  if (deviceId) video.deviceId = { exact: deviceId };
+  else video.facingMode = "user";
+  return video;
+}
+
+async function getCameraBubbleStream(deviceId: string): Promise<MediaStream> {
+  try {
+    return await navigator.mediaDevices.getUserMedia({
+      video: cameraConstraint(deviceId),
+      audio: false,
+    });
+  } catch (error) {
+    if (!deviceId || !isDeviceUnavailableError(error)) throw error;
+    captureExtensionError(
+      new Error("Selected Clips camera was unavailable; using default camera."),
+      {
+        tags: { surface: "overlay", mechanism: "camera-device-fallback" },
+        extra: {
+          requestedDeviceId: deviceId,
+          originalError:
+            error instanceof Error
+              ? { name: error.name, message: error.message }
+              : String(error),
+        },
+      },
+    );
+    return navigator.mediaDevices.getUserMedia({
+      video: cameraConstraint(""),
+      audio: false,
+    });
+  }
+}
+
 function postToolbarSize(height: number): void {
   try {
     window.parent.postMessage(
@@ -161,16 +211,7 @@ async function initBubble(): Promise<void> {
         resolve("");
       }
     });
-    const videoConstraint: MediaTrackConstraints = {
-      width: { ideal: 640 },
-      height: { ideal: 640 },
-    };
-    if (videoDeviceId) videoConstraint.deviceId = { exact: videoDeviceId };
-    else videoConstraint.facingMode = "user";
-    const stream = await navigator.mediaDevices.getUserMedia({
-      video: videoConstraint,
-      audio: false,
-    });
+    const stream = await getCameraBubbleStream(videoDeviceId);
     const video = document.createElement("video");
     video.muted = true;
     video.autoplay = true;