@displayxr/inline3d 1.29.0 → 1.30.0

package/js/camera/capture.js

@@ -0,0 +1,220 @@
+// camera/capture.js — getting a camera (+ microphone) and knowing what it is (RFC §4).
+//
+// `camera: 'auto'` looks for a device that delivers a side-by-side PAIR — one device whose frame
+// is wider than 2.5:1 (a USB stereo camera enumerates as one wide device: field data has one at
+// 1280x480, i.e. 640x480 per eye, grayscale and UNRECTIFIED). Found → sent as `sbs`. Otherwise the
+// default webcam, sent as `mono`. Per-eye width is whatever the camera gives; nothing is upscaled.
+//
+// Two field facts shape the probing:
+//  - On some laptops the stereo camera is held EXCLUSIVELY by the eye tracker, so opening it
+//    fails with NotReadableError ("Device in use"). That device is skipped silently — a call must
+//    never fail because an optional camera is busy.
+//  - Labels are empty until one getUserMedia has succeeded, so the default camera is opened first
+//    (which is also the fallback), then the others are probed one at a time.
+//
+// RECTIFICATION is not done here (P2): it needs the camera's calibration (intrinsics/extrinsics
+// for the ACTIVE device), which comes from a plug-in or the runtime. P1 sends the raw pair with
+// `rectified: false`, and the `rectify` hook (addCall option) is where a calibrated step plugs in:
+// `(stream, info) => MediaStream | Promise<MediaStream>`, returning a rectified SBS stream.
+
+import { looksSbs } from './geometry.js';
+
+const MONO_CONSTRAINTS = { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } };
+// Ask a candidate stereo device for its widest mode; a 2560x720 or 3840x1080 pair comes back as
+// itself, a 1280x480 one as 1280x480. `ideal`, never `min`: a smaller real pair is still a pair.
+const PROBE_CONSTRAINTS = { width: { ideal: 3840 }, height: { ideal: 1080 }, frameRate: { ideal: 30 } };
+export const AUDIO_CONSTRAINTS = Object.freeze({ echoCancellation: true, noiseSuppression: true, autoGainControl: true });
+
+/** Errors that mean "the device exists but another process holds it" (an eye tracker, say). */
+export function isBusyError(err) {
+  return !!err && (err.name === 'NotReadableError' || err.name === 'TrackStartError' || /in use|busy/i.test(err.message || ''));
+}
+
+/** Errors that mean "the user (or a policy) refused this page the camera". */
+export function isDeniedError(err) {
+  return !!err && (err.name === 'NotAllowedError' || err.name === 'PermissionDeniedError' || err.name === 'SecurityError');
+}
+
+/**
+ * Why no camera could be opened, from the per-device failures. `'camera-busy'` when every device
+ * that exists is held by another process — on a 3D laptop/monitor whose only camera is the eye
+ * tracker's, that is the normal case while tracking, and the call should go audio-only and say so
+ * rather than fail or send an empty picture. `'permission-denied'` when any attempt was refused
+ * (a denied prompt, a policy, an insecure context's SecurityError); else `'no-camera'`. Pure.
+ * @param {Array<{error: string, busy?: boolean, denied?: boolean}>} skipped
+ */
+export function noCameraCode(skipped) {
+  if (skipped.length && skipped.every((s) => s.busy)) return 'camera-busy';
+  if (skipped.some((s) => s.denied)) return 'permission-denied';
+  return 'no-camera';
+}
+
+/** A label that names a stereo camera, tried first when probing. Pure. */
+export function stereoLabelHint(label) {
+  return /stereo|\b3d\b|\bsbs\b|dual/i.test(label || '');
+}
+
+/**
+ * Open the camera. Never rejects for a busy optional device; rejects only when NO camera can be
+ * opened at all (the caller then runs audio-only or reports the error).
+ *
+ * @param {'auto'|'stereo'|'mono'|string|MediaStream} want  'auto' | 'stereo' | 'mono' | a
+ *        deviceId | a page-supplied MediaStream
+ * @param {{ format?: 'sbs'|'mono', calibration?: {baselineMm?: number, hfovDeg?: number, rectified?: boolean},
+ *           mediaDevices?: MediaDevices, log?: Function }} [o]
+ * @returns {Promise<{ stream: MediaStream, format: 'sbs'|'mono', width: number, height: number,
+ *           deviceId: string|null, label: string, owned: boolean, skipped: Array<{label: string, error: string}>,
+ *           calibration: object }>}
+ */
+export async function openCamera(want = 'auto', o = {}) {
+  const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
+  const log = o.log || (() => {});
+  const skipped = [];
+  const cal = o.calibration || {};
+
+  if (want && typeof want === 'object' && typeof want.getVideoTracks === 'function') {
+    const t = want.getVideoTracks()[0];
+    const st = t ? t.getSettings() : {};
+    // A page-supplied stream carries a DECLARED format; aspect is never used to guess 3D-ness here.
+    const format = o.format === 'sbs' ? 'sbs' : 'mono';
+    return { stream: want, format, width: st.width || 0, height: st.height || 0, deviceId: st.deviceId || null, label: t ? t.label : '', owned: false, skipped, calibration: cal };
+  }
+  if (!md || !md.getUserMedia) throw Object.assign(new Error('getUserMedia is not available (insecure context?)'), { code: 'no-camera' });
+
+  const open = async (constraints) => {
+    const s = await md.getUserMedia({ video: constraints, audio: false });
+    const t = s.getVideoTracks()[0];
+    const st = t ? t.getSettings() : {};
+    // Never hand back a 0x0 track: some engines open a device that then delivers nothing.
+    if (!t || t.readyState === 'ended' || !(st.width > 0) || !(st.height > 0)) {
+      s.getTracks().forEach((x) => x.stop());
+      throw Object.assign(new Error('the camera opened but delivers no picture (0x0)'), { name: 'NotReadableError' });
+    }
+    // The DisplayXR Browser's "3D Camera (DisplayXR)" (the runtime's stereo camera, e.g. an SR
+    // eye-tracking camera shared without stealing it) describes itself on the opened track:
+    // `displayxrStereo: {layout, rectified, baselineMm, horizontalFovDeg}`.
+    const stereo = st.displayxrStereo && st.displayxrStereo.layout === 'side-by-side' ? st.displayxrStereo : null;
+    return { stream: s, track: t, width: st.width, height: st.height, deviceId: st.deviceId || null, label: t.label || '', stereo };
+  };
+  const skip = (label, err) => {
+    skipped.push({ label, error: `${err.name}: ${err.message}`, busy: isBusyError(err), denied: isDeniedError(err) });
+    log('camera-skip', { label, error: err.name });
+  };
+  const stop = (r) => r && r.stream.getTracks().forEach((t) => t.stop());
+  const result = (r, format) => ({
+    stream: r.stream, format, width: r.width, height: r.height, deviceId: r.deviceId, label: r.label, owned: true, skipped,
+    calibration: {
+      rectified: false,
+      // A runtime stereo camera's own description, where the page gave none.
+      ...(format === 'sbs' && r.stereo
+        ? {
+            rectified: !!r.stereo.rectified,
+            ...(r.stereo.baselineMm > 0 ? { baselineMm: r.stereo.baselineMm } : {}),
+            ...(r.stereo.horizontalFovDeg > 0 ? { hfovDeg: r.stereo.horizontalFovDeg } : {}),
+          }
+        : {}),
+      ...cal,
+    },
+  });
+
+  // An explicit device id.
+  if (typeof want === 'string' && !['auto', 'stereo', 'mono'].includes(want)) {
+    let r;
+    try {
+      r = await open({ deviceId: { exact: want }, ...PROBE_CONSTRAINTS });
+    } catch (err) {
+      skip(want, err);
+      throw Object.assign(new Error(`camera unavailable: ${err.message}`), { code: noCameraCode(skipped), skipped });
+    }
+    if (looksSbs(r.width, r.height)) return result(r, 'sbs');
+    stop(r);
+    return result(await open({ deviceId: { exact: want }, ...MONO_CONSTRAINTS }), 'mono');
+  }
+
+  // Fast path: when labels are already visible (camera permission persisted), open a device that
+  // NAMES itself stereo — e.g. the DisplayXR Browser's "3D Camera (DisplayXR)" — directly. Every
+  // camera start can take seconds (a busy capture stack took 7-20 s per start on a field box), and
+  // the default-first probe below costs two or three of them before it reaches the pair.
+  const tried = new Set();
+  if (want !== 'mono') {
+    let early = [];
+    try {
+      early = (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput' && d.deviceId && stereoLabelHint(d.label));
+    } catch {
+      /* no enumeration yet: the normal path below */
+    }
+    for (const d of early) {
+      tried.add(d.deviceId);
+      let r = null;
+      try {
+        r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
+      } catch (err) {
+        skip(d.label, err);
+        continue;
+      }
+      if (looksSbs(r.width, r.height)) {
+        log('camera-sbs', { label: r.label, width: r.width, height: r.height, direct: true });
+        return result(r, 'sbs');
+      }
+      stop(r);
+    }
+  }
+
+  // The default camera first: permission, labels, and the fallback.
+  let mono = null;
+  try {
+    mono = await open(MONO_CONSTRAINTS);
+  } catch (err) {
+    skip('default', err);
+  }
+  if (want === 'mono' && mono) return result(mono, 'mono');
+  if (mono && looksSbs(mono.width, mono.height)) return result(mono, 'sbs'); // the default IS the pair
+
+  let devices = [];
+  try {
+    devices = (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput');
+  } catch {
+    /* no enumeration: keep the default */
+  }
+  devices.sort((a, b) => Number(stereoLabelHint(b.label)) - Number(stereoLabelHint(a.label)));
+  for (const d of devices) {
+    if (!d.deviceId) continue; // no permission yet: an anonymous entry cannot be opened by id
+    if (mono && d.deviceId === mono.deviceId) continue;
+    if (tried.has(d.deviceId)) continue; // the fast path already opened it (not a pair, or busy)
+    let r = null;
+    try {
+      r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
+    } catch (err) {
+      // NotReadableError = held by another process (e.g. an eye tracker). Skip, never fail.
+      skip(d.label, err);
+      continue;
+    }
+    if (looksSbs(r.width, r.height)) {
+      stop(mono);
+      log('camera-sbs', { label: r.label, width: r.width, height: r.height });
+      return result(r, 'sbs');
+    }
+    stop(r);
+  }
+  if (mono) {
+    if (want === 'stereo') log('camera-no-stereo', { fallback: mono.label });
+    return result(mono, 'mono');
+  }
+  const code = noCameraCode(skipped);
+  throw Object.assign(
+    new Error(code === 'camera-busy' ? 'every camera is in use by another app (e.g. eye tracking)' : code === 'permission-denied' ? 'camera access was denied' : 'no camera could be opened'),
+    { code, skipped }
+  );
+}
+
+/** The microphone, with echo cancellation and noise suppression. Null if unavailable/denied. */
+export async function openMic(o = {}) {
+  const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
+  try {
+    const s = await md.getUserMedia({ audio: { ...AUDIO_CONSTRAINTS }, video: false });
+    return s.getAudioTracks()[0] || null;
+  } catch (err) {
+    (o.log || (() => {}))('mic-skip', { error: err && err.name });
+    return null;
+  }
+}

package/js/camera/converge.js

@@ -0,0 +1,82 @@
+// camera/converge.js — the convergence state of a side-by-side tile (RFC 0002 §3): the per-eye
+// shift that puts a subject at the display plane, from a MEASURED disparity (camera/disparity.js),
+// a sender's hint (focal length · baseline / distance), and the page's depth offset, low-passed
+// per painted frame. Shared by `/camera` (the self view, capture metadata) and `/call` (remote
+// tiles). Pure — see test/call.test.mjs and test/call-autoconv.test.mjs.
+
+/** Low-pass factor for the convergence shift (RFC §3): next = prev + α·(target − prev). */
+export const CONVERGENCE_ALPHA = 0.2;
+
+/** The convergence shift never exceeds this fraction of the per-eye width, either way. */
+export const CONVERGENCE_MAX_FRACTION = 0.12;
+
+/** `setDepth(v)`: v in [-1, 1] adds v times this fraction of the per-eye width to the shift. */
+export const DEPTH_RANGE_FRACTION = 0.05;
+
+/** Focal length in px of an eye image `eyeWidthPx` wide with horizontal FOV `hfovDeg`. */
+export function focalPx(eyeWidthPx, hfovDeg) {
+  if (!(eyeWidthPx > 0) || !(hfovDeg > 0) || hfovDeg >= 180) return null;
+  return eyeWidthPx / 2 / Math.tan(((hfovDeg / 2) * Math.PI) / 180);
+}
+
+/**
+ * Per-eye horizontal shift, in SOURCE eye pixels, that puts a subject at `subjectZmm` on the
+ * display plane: `f_px · baseline / (2 · subjectZ)`. A parallel stereo camera gives everything a
+ * crossed disparity of `f·B/Z` (full, between the eyes); half of it comes off each eye.
+ *
+ * Positive = the left eye's crop moves RIGHT and the right eye's LEFT (content pushed back).
+ * Returns 0 when anything it needs is unknown — the receiver then shows the pair as sent.
+ */
+export function convergenceShiftPx({ eyeWidthPx, hfovDeg, baselineMm, subjectZmm }) {
+  const f = focalPx(eyeWidthPx, hfovDeg);
+  if (f === null || !(baselineMm > 0) || !(subjectZmm > 0)) return 0;
+  return (f * baselineMm) / (2 * subjectZmm);
+}
+
+/** Clamp a shift to ±CONVERGENCE_MAX_FRACTION of the per-eye width. */
+export function clampShift(px, eyeWidthPx, maxFraction = CONVERGENCE_MAX_FRACTION) {
+  const lim = Math.max(0, eyeWidthPx * maxFraction);
+  return Math.max(-lim, Math.min(lim, px || 0));
+}
+
+/** One low-pass step: `prev + α(target − prev)`. Snaps when within 0.05 px so it can settle. */
+export function lowPass(prev, target, alpha = CONVERGENCE_ALPHA) {
+  const next = prev + alpha * (target - prev);
+  return Math.abs(target - next) < 0.05 ? target : next;
+}
+
+/**
+ * A tile's convergence state: the target from hello+hint, the page's depth offset, and the
+ * smoothed value actually painted. `step()` once per painted frame.
+ */
+export function createConvergence() {
+  const s = {
+    hello: null,
+    subjectZmm: null,
+    // Auto-convergence (call/disparity.js): the MEASURED disparity of the point between the remote
+    // person's eyes, in source eye pixels (left x − right x). When set it wins over the hint: it is
+    // read off the frames, so it needs no calibration and cannot disagree with them.
+    measuredPx: null,
+    depth: 0, // setDepth(), [-1, 1]
+    current: 0,
+    target(eyeWidthPx) {
+      const auto =
+        s.measuredPx !== null
+          ? s.measuredPx / 2 // half the disparity comes off each eye: the eyes land at zero parallax
+          : s.hello && s.subjectZmm
+            ? convergenceShiftPx({
+                eyeWidthPx,
+                hfovDeg: s.hello.hfovDeg,
+                baselineMm: s.hello.baselineMm,
+                subjectZmm: s.subjectZmm,
+              })
+            : 0;
+      return clampShift(auto + s.depth * DEPTH_RANGE_FRACTION * eyeWidthPx, eyeWidthPx);
+    },
+    step(eyeWidthPx) {
+      s.current = lowPass(s.current, s.target(eyeWidthPx));
+      return s.current;
+    },
+  };
+  return s;
+}