@displayxr/inline3d 1.28.0 → 1.29.0

package/js/inline3d-call.js

@@ -1,17 +1,21 @@
 // inline3d-call.js — a 3D video call in any page, in one call.
 //
 // PREVIEW tier. Not covered by the SDK's 1.x semver promise — see docs/sdk-stability.md.
-// Implements P1 of docs/rfcs/0002-video-call.md.
+// Implements P1 of docs/rfcs/0002-video-call.md and the one-line path of RFC 0003 §1 (C1).
 //
-//   import { createInline3D } from '@displayxr/inline3d';
-//   import { addCall, dxrSignaling } from '@displayxr/inline3d/call';
-//
-//   const wall = await createInline3D();
-//   const call = await addCall(wall, document.getElementById('call'), {
-//     signaling: dxrSignaling(),            // hosted DisplayXR server; or dxrSignaling('wss://your.server')
-//   });
+//   import { mountCall } from '@displayxr/inline3d/call';
+//   const call = await mountCall(document.getElementById('call'));   // hosted signalling, auto camera, SDK UI
 //   call.on('peer', ({ id }) => console.log('joined', id));
 //
+// `mountCall(el, opts?)` is `addCall` with every argument optional: the wall is the document's
+// shared manager (`sharedInline3D()` — never a second session next to the page's own), signalling
+// defaults to the hosted server, the camera to 'auto', the chrome to on. `addCall(wall, container,
+// opts)` stays as the explicit form for a page that manages its own wall. `<dxr-call>`
+// (call/element.js) is the same thing as markup. A bundled app that wants 2D→3D imports
+// `@displayxr/inline3d/call/full` (lift statically imported) or passes `mono3D: lift` itself —
+// on the plain `/call` entry a mono peer on a 3D display raises `warning { code:
+// 'lift-not-bundled' }` instead of silently staying flat.
+//
 // WHAT IT OWNS. Unlike the other modules this one takes a CONTAINER, not a canvas: a call has a
 // variable number of participants, so it creates one persistent canvas per remote participant
 // (plus the self view) inside `container`, lays them out, and draws its own chrome around them.
@@ -67,7 +71,7 @@
 // SDK's teardown synchronously, so no listener can distinguish them afterwards); `leave()`
 // restores it.
 
-import { createInline3D } from './inline3d.js';
+import { sharedInline3D } from './inline3d.js';
 import {
   WIRE_VERSION,
   CALL_SDK,
@@ -97,9 +101,12 @@ import { openCamera, openMic } from './call/capture.js';
 import { drawQr } from './call/qr.js';
 import { injectCallStyle, ICONS, el, show, resolveCallAccent, CALL_ACCENTS } from './call/ui.js';
 import { normalizeMono3D, resolveLift, createLiftPool, createFrameWatch } from './call/lift.js';
+import { defineCallElement } from './call/element.js';
+export { DxrCallElement, defineCallElement, attrsToOpts, CALL_EVENT_PREFIX } from './call/element.js';
 
 import { dxrSignaling } from './call/signaling.js';
-export { dxrSignaling, peerjsCloud, SIGNAL_PROTOCOL, roomKey, DXR_SIGNAL_DEFAULT } from './call/signaling.js';
+import { lumaFromRgba, createDisparityTrack, createFocusTracker, downsampleLuma } from './call/disparity.js';
+export { dxrSignaling, SIGNAL_PROTOCOL, roomKey, DXR_SIGNAL_DEFAULT } from './call/signaling.js';
 export {
   WIRE_VERSION,
   CALL_SDK,
@@ -114,6 +121,7 @@ export {
   CALL_ACCENTS,
 };
 export { createLiveGate } from './call/wire.js';
+export { measureFocusDisparity } from './call/disparity.js';
 export { convergenceShiftPx, lowPass, clampShift, eyeCropRect, mirrorSwapOps, mirrorSwapPixels, maxBitrateKbps } from './call/wire.js';
 export { preferVideoCodecs, sortCodecCapabilities, VIDEO_CODEC_ORDER } from './call/sdp.js';
 export { MeshTransport, clampMaxPeers } from './call/transport.js';
@@ -137,6 +145,12 @@ const STATS_TICK_MS = 500;
 const SPEAKING_LEVEL = 0.02;
 const LAYER_RETRIES = 4;
 const LIFT_RETRIES = 2;
+// Auto-convergence sampling: ~5 Hz on a copy whose eye is at most this wide (enough for sub-pixel
+// disparity on a face; small enough to stay a few ms per measurement).
+const AUTO_CONV_INTERVAL_MS = 200;
+const AUTO_CONV_EYE_WIDTH = 240;
+// WebCodecs pixel formats whose plane 0 is luma (Y): read directly, no colour conversion.
+const Y_PLANE_FORMATS = new Set(['I420', 'I420A', 'I422', 'I444', 'NV12']);
 const DEFAULT_BROWSER_URL = 'https://github.com/DisplayXR/displayxr-browser';
 export const PLATE_TEXT = Object.freeze({
   unreachable: "Can't reach this participant — the network needs a relay (TURN)",
@@ -158,14 +172,19 @@ export function normalizeCallOptions(opts = {}) {
   }
   return {
     room,
-    // Default: the hosted DisplayXR signalling server (which also mints TURN credentials).
-    signaling: opts.signaling || dxrSignaling(),
+    // Default: the hosted DisplayXR signalling server (which also mints TURN credentials). A
+    // publishable key (RFC 0003 §5a) rides along on connect; the server may ignore it today.
+    signaling: opts.signaling || dxrSignaling(undefined, typeof opts.key === 'string' && opts.key ? { key: opts.key } : {}),
     iceServers: Array.isArray(opts.iceServers) ? opts.iceServers : undefined,
     camera: opts.camera === undefined ? 'auto' : opts.camera,
     format: opts.format === 'sbs' ? 'sbs' : opts.format === 'mono' ? 'mono' : undefined,
     calibration: opts.calibration && typeof opts.calibration === 'object' ? { ...opts.calibration } : {},
     rectify: typeof opts.rectify === 'function' ? opts.rectify : null,
     audio: opts.audio === undefined ? true : !!opts.audio,
+    // Auto-convergence (call/disparity.js): measure the disparity of the point between each SBS
+    // peer's eyes and shift the eyes so it sits at the display plane. The depth slider stays an
+    // offset on top. Off = the pair as sent (plus any `hint`).
+    autoConverge: opts.autoConverge === undefined ? true : !!opts.autoConverge,
     mono3D: normalizeMono3D(opts.mono3D),
     maxPeers: clampMaxPeers(opts.maxPeers === undefined ? DEFAULT_MAX_PEERS : opts.maxPeers),
     // Extra lift() options for lifted tiles (models, ort, quality, providers). The call's own
@@ -186,6 +205,8 @@ export function normalizeCallOptions(opts = {}) {
     scrollIntoView: opts.scrollIntoView === undefined ? true : !!opts.scrollIntoView,
     wallOptions: opts.wallOptions && typeof opts.wallOptions === 'object' ? opts.wallOptions : {},
     log: typeof opts.log === 'function' ? opts.log : opts.debug ? (tag, obj) => console.log(`${TAG} ${tag} ${JSON.stringify(obj)}`) : null,
+    // Internal (tests): replaces the lift module importer behind `mono3D: 'auto'`.
+    _liftImporter: typeof opts._liftImporter === 'function' ? opts._liftImporter : undefined,
   };
 }
 
@@ -201,13 +222,30 @@ export async function addCall(wall, container, opts = {}) {
   if (!container || typeof container.appendChild !== 'function') throw new TypeError('@displayxr/inline3d/call: addCall(wall, container, opts) needs a container element');
   const o = normalizeCallOptions(opts);
   if (!o.signaling || typeof o.signaling.join !== 'function') {
-    throw new TypeError('@displayxr/inline3d/call: opts.signaling must be a SignalingAdapter — dxrSignaling([url]), peerjsCloud() (demo only), or your own');
+    throw new TypeError('@displayxr/inline3d/call: opts.signaling must be a SignalingAdapter — dxrSignaling([url]) or your own');
   }
   const call = new Call(wall, container, o);
   await call._init();
   return call.handle;
 }
 
+/**
+ * The one-line path (RFC 0003 §1): `addCall` with every argument optional. The wall is
+ * `opts.wall` if given, else the document's shared manager — `sharedInline3D(opts.wallOptions)`,
+ * which returns the page's existing session rather than opening a second one (woven-canvas
+ * rule 1). Everything else defaults as `addCall` does: hosted signalling, `camera: 'auto'`,
+ * `ui: true`.
+ *
+ * @param {HTMLElement} el  the container (a `<dxr-call>`, a `<div>`, …)
+ * @param {object} [opts]   see call.d.ts `CallOptions`
+ */
+export async function mountCall(el, opts = {}) {
+  if (!el || typeof el.appendChild !== 'function') throw new TypeError('@displayxr/inline3d/call: mountCall(el, opts) needs a container element');
+  const { wall: given, ...rest } = opts;
+  const wall = given || (await sharedInline3D(rest.wallOptions));
+  return addCall(wall, el, rest);
+}
+
 class Call {
   constructor(wall, container, o) {
     this.o = o;
@@ -238,6 +276,8 @@ class Call {
     this.liftPool = null;
     this._liftP = null;
     this.liftProven = false;
+    this.liftMissing = false; // the lift MODULE failed to import (vs. a display that can't lift)
+    this._liftWarned = false;
     this._frameWatch = createFrameWatch();
     this.handle = this._makeHandle();
     if (this.wallLive) this._hookWall(this.wall);
@@ -272,6 +312,13 @@ class Call {
     this.emit('error', { code, message, error: err || null });
   }
 
+  /** Degraded, not broken (RFC 0003 §2): one console.warn (always — the page must see it) + an event. */
+  warning(code, message) {
+    this.log('warning', { code, message });
+    console.warn(`${TAG} ${code}: ${message}`);
+    this.emit('warning', { code, message });
+  }
+
   // ── setup ────────────────────────────────────────────────────────────────────────────────
 
   async _init() {
@@ -695,7 +742,9 @@ class Call {
       if (this.state === 'left') return;
       let w = null;
       try {
-        w = await createInline3D(this.o.wallOptions);
+        // The shared manager: if the page has already re-opened its own wall, join it rather than
+        // opening a second session (rule 1); else this creates the document's new one.
+        w = await sharedInline3D(this.o.wallOptions);
       } catch {
         w = null;
       }
@@ -748,7 +797,7 @@ class Call {
       const lo = this.o.liftOptions;
       // liftCapabilities probes the same model source the lifts will use.
       const capsOpts = { webFallback: true, ...(lo && lo.models ? { models: lo.models } : {}) };
-      this._liftP = resolveLift(this.mono3D, { log, capsOpts }).then((r) => {
+      this._liftP = resolveLift(this.mono3D, { log, capsOpts, ...(this.o._liftImporter ? { importer: this.o._liftImporter } : {}) }).then((r) => {
         this.liftApi = r;
         if (r.lift) {
           this.liftPool = createLiftPool({ lift: r.lift, max: this.o.maxLifted, log, options: this.o.liftOptions });
@@ -763,6 +812,12 @@ class Call {
               })
               .catch(() => {});
           }
+        } else if (r.reason === 'import-failed' || r.reason === 'no-lift-export') {
+          // The MODULE is missing (a bundled app whose bundler could not follow the computed
+          // import, or a copy of the SDK built without lift) — not a display that can't lift. Said
+          // loudly, once, when a mono peer actually arrives (_warnLiftMissing): a developer testing
+          // in the DisplayXR Browser sees it the first time a 2D caller joins.
+          this.liftMissing = true;
         } else if (r.reason !== 'off') {
           console.info(`${TAG} mono→3D unavailable (${r.reason}) — mono participants are shown flat`);
         }
@@ -774,6 +829,21 @@ class Call {
     return this._liftP;
   }
 
+  /**
+   * `warning { code: 'lift-not-bundled' }`, once per call: a mono peer is on a 3D display, lift
+   * was asked for ('auto'), and the lift module could not be imported. Names the fix.
+   */
+  _warnLiftMissing() {
+    if (this._liftWarned || !this.liftMissing || !this.mono3DOn) return;
+    this._liftWarned = true;
+    this.warning(
+      'lift-not-bundled',
+      'a 2D participant is on a 3D display but the 2D→3D module could not be imported from this build, so they stay flat. ' +
+        "Import it explicitly — `import { lift } from '@displayxr/inline3d/lift'; mountCall(el, { mono3D: lift })` — " +
+        "or use `@displayxr/inline3d/call/full`, which bundles it."
+    );
+  }
+
   /** Re-run the routing table for every mono tile (lift resolved, a slot freed, setMono3D). */
   _rerouteMono() {
     for (const t of this.tiles.values()) if (t.format && t.format !== 'sbs') t.reroute(false);
@@ -798,7 +868,7 @@ class Call {
     if (!this.mono3DOn) return 'Mono cameras: 2D (2D→3D is off).';
     const r = this.liftApi;
     if (!r) return 'Mono cameras: checking 2D→3D…';
-    if (!r.lift) return 'Mono cameras: 2D (no 2D→3D provider here).';
+    if (!r.lift) return this.liftMissing ? 'Mono cameras: 2D→3D unavailable in this build.' : 'Mono cameras: 2D (no 2D→3D provider here).';
     const c = r.caps;
     if (c && c.native) return `Mono cameras: 2D→3D (native${c.provider ? `, ${c.provider}` : ''}).`;
     if (c && c.webFallback && c.webFallback.webgpu) return 'Mono cameras: 2D→3D (in this page, WebGPU).';
@@ -1219,6 +1289,17 @@ class Tile {
     this.speaking = false;
     this.quality = null;
     this.conv = createConvergence();
+    this.autoTrack = createDisparityTrack();
+    this.autoFocus = createFocusTracker();
+    this.autoAt = 0;
+    this.autoCanvas = null;
+    this.autoLocked = false;
+    this.autoStats = { n: 0, ms: 0, readMs: 0, method: null, via: null };
+    this.autoPending = false;
+    // Bumped on every stream change and on destroy(): an async copyTo() that resolves after either
+    // belongs to a stream this tile no longer shows, and is dropped (review of #92).
+    this.autoGen = 0;
+    this.destroyed = false;
     this.conv.depth = call.depth;
     this.hintGate = rateGate(HINT_MAX_HZ);
     this.layerFails = 0;
@@ -1263,6 +1344,15 @@ class Tile {
       hello: this.hello,
       quality: this.quality,
       convergencePx: this.conv.current,
+      // Auto-convergence: the measured disparity (source px, null until a lock) and the mean cost of
+      // one measurement including the frame readback.
+      autoConverge: {
+        disparityPx: this.conv.measuredPx,
+        ms: this.autoStats.n ? this.autoStats.ms / this.autoStats.n : null,
+        readMs: this.autoStats.n ? this.autoStats.readMs / this.autoStats.n : null,
+        method: this.autoStats.method,
+        via: this.autoStats.via,
+      },
       lift: this.lifted
         ? Object.freeze({
             live: this.liftLive,
@@ -1278,6 +1368,7 @@ class Tile {
     if (this.video.srcObject !== stream) {
       this.video.srcObject = stream;
       this.audio.srcObject = stream;
+      this._resetAutoConverge(); // a new stream may be a new camera: measure afresh
     }
     this.video.play().catch(() => {});
     this.audio.play().catch(() => {});
@@ -1383,6 +1474,8 @@ class Tile {
           });
     // First mono peer on a woven wall and lift not asked for yet (e.g. setMono3D(true) later).
     if (this.format !== 'sbs' && call.woven && call.mono3DOn && !call._liftP) call._ensureLift();
+    // A mono peer on a 3D display that stays flat because the lift MODULE is missing: say so.
+    if (this.format !== 'sbs' && call.woven && call.liftMissing) call._warnLiftMissing();
     const key = `${r.route}|${this.format}`;
     if (!force && key === this.routedKey) return;
     this.routedKey = key;
@@ -1572,6 +1665,7 @@ class Tile {
       c.width = 2 * outW;
       c.height = outH;
     }
+    if (this.call.o.autoConverge) this._sampleDisparity(v, W, H);
     const shift = this.conv.step(eyeW);
     const g = c.getContext('2d');
     for (const eye of [0, 1]) {
@@ -1580,7 +1674,114 @@ class Tile {
     }
   }
 
+  /**
+   * Auto-convergence: a few times a second, measure the disparity of the point between the remote
+   * person's eyes on a small grayscale copy of the SOURCE frame (before our shift, so there is no
+   * feedback loop) and hand it to the convergence state, which halves it per eye and low-passes it.
+   * A failed measurement holds the last good value.
+   */
+  _sampleDisparity(v, W, H) {
+    const now = performance.now();
+    if (this.autoPending || now - this.autoAt < AUTO_CONV_INTERVAL_MS) return;
+    this.autoAt = now;
+    // Preferred: WebCodecs. `new VideoFrame(video)` + an ASYNC copyTo() of the decoded frame, reading
+    // the luma (Y) plane directly: no synchronous GPU->CPU canvas readback on the paint path (that
+    // readback, not the matching, was ~25 ms of a ~30 ms measurement).
+    if (typeof VideoFrame === 'function') {
+      let frame = null;
+      try {
+        frame = new VideoFrame(v);
+      } catch {
+        frame = null;
+      }
+      if (frame && Y_PLANE_FORMATS.has(frame.format)) {
+        this.autoPending = true;
+        const gen = this.autoGen;
+        const fw = frame.visibleRect ? frame.visibleRect.width : frame.displayWidth;
+        const fh = frame.visibleRect ? frame.visibleRect.height : frame.displayHeight;
+        const buf = new Uint8Array(frame.allocationSize());
+        frame
+          .copyTo(buf)
+          .then((layout) => {
+            if (gen !== this.autoGen) return; // the stream changed or the tile is gone: stale frame
+            const t0 = performance.now();
+            const f = Math.max(1, Math.ceil(fw / 2 / AUTO_CONV_EYE_WIDTH));
+            const ds = downsampleLuma(buf, layout[0].offset, layout[0].stride, fw, fh, f);
+            this._applyMeasurement(ds.img, ds.w, ds.h, ds.w / fw, now, performance.now() - t0, 'videoframe');
+          })
+          .catch(() => {
+            if (gen === this.autoGen) this._applyMeasurement(null, 0, 0, 1, now, 0, 'videoframe');
+          })
+          .finally(() => {
+            frame.close();
+            this.autoPending = false;
+          });
+        return;
+      }
+      if (frame) frame.close();
+    }
+    // Fallback: a small canvas + getImageData (synchronous readback).
+    const s = Math.min(1, AUTO_CONV_EYE_WIDTH / (W / 2));
+    const w = Math.max(2, Math.round((W * s) / 2) * 2);
+    const h = Math.max(2, Math.round(H * s));
+    if (!this.autoCanvas) this.autoCanvas = document.createElement('canvas');
+    const c = this.autoCanvas;
+    if (c.width !== w || c.height !== h) {
+      c.width = w;
+      c.height = h;
+    }
+    const g = c.getContext('2d', { willReadFrequently: true });
+    let img = null;
+    try {
+      g.drawImage(v, 0, 0, w, h);
+      img = lumaFromRgba(g.getImageData(0, 0, w, h).data, w * h);
+    } catch {
+      img = null; // a frame that cannot be read (tainted, not decoded yet): skip it
+    }
+    this._applyMeasurement(img, w, h, w / W, now, performance.now() - now, 'canvas');
+  }
+
+  /**
+   * Measure one grabbed frame and hand the disparity to the convergence state (which halves it per
+   * eye and low-passes it). A failed measurement holds the last good value. `readMs` = main-thread
+   * time spent getting the pixels.
+   */
+  _applyMeasurement(img, w, h, s, at, readMs, via) {
+    if (this.destroyed) return;
+    const t = performance.now();
+    const m = img ? this.autoFocus.measure(img, w, h, at) : null;
+    const d = this.autoTrack.push(m ? m.d / s : null);
+    this.conv.measuredPx = d;
+    this.autoStats.n++;
+    this.autoStats.readMs += readMs;
+    this.autoStats.ms += readMs + (performance.now() - t);
+    this.autoStats.method = m ? m.method : null;
+    this.autoStats.via = via;
+    if (m && !this.autoLocked) {
+      this.autoLocked = true;
+      this.call.log('auto-converge', {
+        peer: this.id,
+        disparityPx: Math.round(d * 10) / 10,
+        at: { x: Math.round(m.x / s), y: Math.round(m.y / s) },
+        ncc: Math.round(m.c * 100) / 100,
+        method: m.method,
+        via,
+        ms: Math.round(readMs + (performance.now() - t)),
+      });
+    }
+  }
+
+  _resetAutoConverge() {
+    this.autoGen++;
+    this.autoTrack.reset();
+    this.autoFocus.reset();
+    this.conv.measuredPx = null;
+    this.autoLocked = false;
+  }
+
   destroy() {
+    this.destroyed = true;
+    this.autoGen++;
     clearTimeout(this.helloTimer);
     clearTimeout(this.layerTimer);
     clearTimeout(this.leaveTimer);
@@ -1693,3 +1894,8 @@ class SelfTile {
     }
   }
 }
+
+// `<dxr-call>` — the one-line path as markup — is registered by importing this entry (RFC 0003 §1;
+// the ONE side effect of this module, declared in package.json `sideEffects`). A no-op where there
+// is no `customElements` registry (Node, workers) or the name is already defined.
+defineCallElement();