@displayxr/inline3d 1.39.0 → 1.40.0

package/js/call/element.js

@@ -71,6 +71,7 @@ const str = (v) => (v === null || v === undefined || String(v).trim() === '' ? u
  * | `no-audio`           | `audio: false`                           |
  * | `no-self-view`       | `selfView: false`                        |
  * | `no-auto-converge`   | `autoConverge: false`                    |
+ * | `cursor="depth"`     | `cursor: 'depth'`                        |
  * | `tile-aspect`        | `theme.tileAspect` (a number, or `w/h`)  |
  * | `invite-base`        | `invite.base`                            |
  * | `browser-url`        | `landing.browserUrl`                     |
@@ -101,6 +102,7 @@ export function attrsToOpts(source) {
   if (isOn(get('no-audio'))) o.audio = false;
   if (isOn(get('no-self-view'))) o.selfView = false;
   if (isOn(get('no-auto-converge'))) o.autoConverge = false;
+  if (str(get('cursor')) === 'depth') o.cursor = 'depth';
   // The C2 groups (RFC 0003 §2): the attribute names are the C1 ones, the option shapes are new.
   const theme = { accent: str(get('accent')), tileAspect: num(get('tile-aspect')) };
   if (theme.accent !== undefined || theme.tileAspect !== undefined) o.theme = Object.fromEntries(Object.entries(theme).filter(([, v]) => v !== undefined));

package/js/call/options.js

@@ -8,6 +8,7 @@ import { clampMaxPeers } from './transport.js';
 import { normalizeMono3D } from './lift.js';
 import { dxrSignaling } from './signaling.js';
 import { resolveCallAccent, THEME_VARS } from './ui.js';
+import { resolvePairCursorOption } from '../inline3d-cursor-option.js';
 
 const TAG = '[inline3d/call]';
 const DEFAULT_BROWSER_URL = 'https://github.com/DisplayXR/displayxr-browser';
@@ -89,6 +90,9 @@ export function normalizeCallOptions(opts = {}) {
     // peer's eyes and shift the eyes so it sits at the display plane. The depth slider stays an
     // offset on top. Off = the pair as sent (plus any `hint`).
     autoConverge: opts.autoConverge === undefined ? true : !!opts.autoConverge,
+    // `cursor: 'depth'` (runtime ADR-046 on a pair): a crosshair just in front of the person under
+    // the pointer, measured off each stereo tile's two halves. Off by default (zero cost).
+    cursor: resolvePairCursorOption(opts.cursor, '@displayxr/inline3d/call'),
     mono3D: normalizeMono3D(opts.mono3D),
     maxPeers: clampMaxPeers(opts.maxPeers === undefined ? DEFAULT_MAX_PEERS : opts.maxPeers),
     // Extra lift() options for lifted tiles (models, ort, quality, providers). The call's own

package/js/camera/pair-cursor.js

@@ -0,0 +1,166 @@
+// camera/pair-cursor.js — the depth-aware cursor on a STEREO PAIR: a call tile or a camera view.
+//
+// The OS cursor is drawn at zero disparity, on the glass. Over a person who sits in front of it
+// (a converged stereo call puts the face AT the glass, so the shoulders, a raised hand, a cup held
+// up are all in front), it is drawn on top of pixels that say they are nearer — the depth
+// violation runtime ADR-046 is about. A pair has no scene to raycast and no depth buffer, but its
+// two halves ARE the depth: this is ADR-046 Phase 3b done in the page. Block-match a cursor-sized
+// footprint between the eyes, take the nearest confident match, and draw a crosshair into both
+// halves with that disparity plus a small margin. The browser weaves the pair as usual, so the
+// sprite lands just in front of the content under it.
+//
+// Same policy as the scene cursor (./inline3d-cursor-depth.js): the footprint ring, the fast-rise /
+// slow-sink filter, the display plane when nothing matches. Disparities here are in OUTPUT pixels
+// of one eye of the woven buffer, + = crossed (in front of the glass), the sign the rest of
+// camera/ uses. Loaded only when a page asks for `cursor: 'depth'` — zero cost otherwise.
+
+import { CursorPointer, cursorFootprint, cursorFilterStep, CURSOR_DEFAULT_TUNING, CURSOR_CROSSHAIR, CURSOR_FILL_COLOR, CURSOR_OUTLINE_COLOR } from '../inline3d-cursor-depth.js';
+import { matchBlock, MIN_NCC } from './disparity.js';
+
+/** Sprite height, as a fraction of the tile height (a call tile is small; 3% would be a speck). */
+export const PAIR_CURSOR_HEIGHT = 0.06;
+/** How far in front of the content the sprite floats, as a fraction of the eye width (crossed). */
+export const PAIR_CURSOR_MARGIN = 0.002;
+/** Comfort clamp on the sprite's disparity, as a fraction of the eye width, either way. */
+export const PAIR_CURSOR_CLAMP = 0.08;
+
+/**
+ * The displayed disparity of the nearest content under the cursor's footprint, or null when no
+ * point of the footprint matched confidently (flat, ambiguous, off the frame).
+ *
+ * @param {{img: ArrayLike<number>, w: number, h: number, scale: number}} luma  a grayscale copy
+ *        of the WHOLE side-by-side source frame; `scale` = copy px per source px.
+ * @param {{rL: {sx:number, sy:number, sw:number, sh:number}, rR: {sx:number}, outW: number, mirror?: boolean}} geo
+ *        each eye's source crop (relative to its half, eyeCropRect) and the output eye width; a
+ *        mirrored view (the self view) flips the pointer, never the disparity.
+ * @param {number} u  pointer, tile-normalised (0..1, left → right as the viewer sees it)
+ * @param {number} v  pointer, tile-normalised (0..1, top → bottom)
+ * @param {number} height  sprite height (fraction of the tile height)
+ * @param {number} aspect  the tile's width / height, for a round footprint
+ * @returns {number|null}  output px, + = in front of the glass
+ */
+export function pairFootprintDisparity(luma, geo, u, v, height, aspect) {
+  if (!luma || !luma.img || !geo || !geo.rL || !geo.rR || !(geo.outW > 0)) return null;
+  const { img, w, h, scale } = luma;
+  const E = w / 2;
+  const b = Math.max(8, Math.round(E / 20));
+  const dMax = Math.round(E * 0.3);
+  const dMin = -Math.round(E * 0.05);
+  const { rL, rR } = geo;
+  const k = geo.outW / rL.sw; // output px per source px
+  let best = null;
+  for (const [pu, pv] of cursorFootprint(u, v, height, aspect)) {
+    if (!(pu >= 0 && pu <= 1 && pv >= 0 && pv <= 1)) continue;
+    const su = geo.mirror ? 1 - pu : pu;
+    const x = Math.round((rL.sx + su * rL.sw) * scale - b / 2);
+    const y = Math.round((rL.sy + pv * rL.sh) * scale - b / 2);
+    if (x < 0 || x + b > E || y < 0 || y + b > h) continue;
+    const m = matchBlock(img, w, h, x, y, b, b, { dMin, dMax, dyMax: 2 });
+    if (!m || m.c < MIN_NCC) continue;
+    // Source disparity → displayed: each eye's crop moved its half by its own offset.
+    const d = (m.d / scale - (rL.sx - rR.sx)) * k;
+    if (best === null || d > best) best = d;
+  }
+  return best;
+}
+
+/** The sprite's target disparity: the content plus the margin, clamped; the glass with nothing under it. */
+export function pairCursorTarget(contentPx, outW) {
+  if (contentPx === null || !Number.isFinite(contentPx)) return 0;
+  const lim = PAIR_CURSOR_CLAMP * outW;
+  return Math.max(-lim, Math.min(lim, contentPx + PAIR_CURSOR_MARGIN * outW));
+}
+
+/**
+ * Draw the crosshair into both halves of a woven buffer: the left eye's copy `disparityPx / 2`
+ * right of the pointer, the right eye's as far left (crossed = in front). Each copy is clipped to
+ * its own half, so a sprite at the edge never bleeds into the other eye.
+ */
+export function drawPairCrosshair(g, outW, outH, u, v, disparityPx, height) {
+  const size = height * outH;
+  const cy = v * outH;
+  const rgba = (c) => `rgba(${Math.round(c[0] * 255)},${Math.round(c[1] * 255)},${Math.round(c[2] * 255)},${c[3]})`;
+  for (const eye of [0, 1]) {
+    const cx = eye * outW + u * outW + (eye === 0 ? disparityPx / 2 : -disparityPx / 2);
+    g.save();
+    g.beginPath();
+    g.rect(eye * outW, 0, outW, outH);
+    g.clip();
+    for (const [width, color, cap] of [
+      [0.15, CURSOR_OUTLINE_COLOR, 'square'],
+      [0.07, CURSOR_FILL_COLOR, 'butt'],
+    ]) {
+      g.lineWidth = width * size;
+      g.lineCap = cap;
+      g.strokeStyle = rgba(color);
+      g.beginPath();
+      for (const [x0, y0, x1, y1] of CURSOR_CROSSHAIR) {
+        g.moveTo(cx + x0 * size, cy + y0 * size);
+        g.lineTo(cx + x1 * size, cy + y1 * size);
+      }
+      g.stroke();
+    }
+    g.restore();
+  }
+}
+
+/**
+ * One cursor over one woven pair. The owner calls `draw()` after painting the pair each frame
+ * (`idle()` on any frame it does not paint a woven pair), and hands it the latest grayscale copy
+ * of the source when it has one; `hovering` says when that copy is worth grabbing more often.
+ */
+export class PairCursor {
+  /** @param {HTMLElement} canvas  the tile — the pointer counts over its box */
+  constructor(canvas, { height, pointerScope = 'window' } = {}) {
+    this.pointer = new CursorPointer(canvas, { scope: pointerScope });
+    this.height = height > 0 ? height : PAIR_CURSOR_HEIGHT;
+    this.filter = {};
+    this._luma = null;
+    this._uv = null;
+    this._content = null;
+    /** The last frame's placement, for diagnostics: `{ contentPx, targetPx, disparityPx }` or null. */
+    this.last = null;
+  }
+
+  get hovering() {
+    return !!this.pointer.uv;
+  }
+
+  /**
+   * @param {CanvasRenderingContext2D} g  the woven buffer (2·outW × outH), already painted
+   * @param {object|null} luma  the latest grayscale source copy (pairFootprintDisparity)
+   * @param {{rL: object, rR: object, outW: number, outH: number, mirror?: boolean}} geo
+   * @param {number} nowSec
+   */
+  draw(g, luma, geo, nowSec) {
+    const uv = this.pointer.uv;
+    if (!uv) return this.idle();
+    const [u, v] = uv;
+    // Match only when there is something new to match: a new frame copy or a moved pointer.
+    if (luma !== this._luma || !this._uv || this._uv[0] !== u || this._uv[1] !== v) {
+      this._content = pairFootprintDisparity(luma, geo, u, v, this.height, geo.outW / geo.outH);
+      this._luma = luma;
+      this._uv = uv;
+    }
+    const target = pairCursorTarget(this._content, geo.outW);
+    // The shared filter rises fast toward SMALLER values (nearer, in its convention): negate.
+    const disparity = -cursorFilterStep(this.filter, CURSOR_DEFAULT_TUNING, -target, nowSec);
+    drawPairCrosshair(g, geo.outW, geo.outH, u, v, disparity, this.height);
+    this.pointer.hideCss(true);
+    this.last = { contentPx: this._content, targetPx: target, disparityPx: disparity };
+    return this.last;
+  }
+
+  /** Not drawing this frame (pointer away, tile not woven): the CSS cursor comes back. */
+  idle() {
+    this.pointer.hideCss(false);
+    this.filter = {};
+    this._uv = null;
+    this.last = null;
+    return null;
+  }
+
+  dispose() {
+    this.pointer.dispose();
+  }
+}

package/js/inline3d-call.js

@@ -117,6 +117,8 @@ const LIFT_RETRIES = 2;
 // disparity on a face; small enough to stay a few ms per measurement).
 const AUTO_CONV_INTERVAL_MS = 200;
 const AUTO_CONV_EYE_WIDTH = 240;
+// While a depth cursor hovers a stereo tile, its source is copied this often (the cursor measures it).
+const CURSOR_SAMPLE_MS = 66;
 // WebCodecs pixel formats whose plane 0 is luma (Y): read directly, no colour conversion.
 const Y_PLANE_FORMATS = new Set(['I420', 'I420A', 'I422', 'I444', 'NV12']);
 /**
@@ -1306,6 +1308,7 @@ class Tile {
     // Bumped on every stream change and on destroy(): an async copyTo() that resolves after either
     // belongs to a stream this tile no longer shows, and is dropped (review of #92).
     this.autoGen = 0;
+    this.luma = null; // the latest grayscale source copy: auto-convergence and the depth cursor read it
     this.destroyed = false;
     this.conv.depth = call.depth;
     this.hintGate = rateGate(HINT_MAX_HZ);
@@ -1337,6 +1340,14 @@ class Tile {
     }
     call.ui.grid.appendChild(this.el);
     this._renderBadge();
+    /** The depth cursor (`cursor: 'depth'`) once its module has loaded; null otherwise. */
+    this.cursor = null;
+    if (call.o.cursor) {
+      call._pairCursorMod = call._pairCursorMod || import('./camera/pair-cursor.js');
+      call._pairCursorMod.then((m) => {
+        if (!this.destroyed) this.cursor = new m.PairCursor(this.canvas, call.o.cursor);
+      });
+    }
   }
 
   /** How this side shows the participant: `3D` (woven pair) | `2D→3D` (lifted) | `2D` (flat). */
@@ -1366,6 +1377,8 @@ class Tile {
       hello: this.hello,
       quality: this.quality,
       convergencePx: this.conv.current,
+      // The depth cursor's last placement (output px, + = in front of the glass), null when not drawn.
+      cursor: this.cursor ? this.cursor.last : null,
       // Auto-convergence: the measured disparity (source px, null until a lock) and the mean cost of
       // one measurement including the frame readback.
       autoConverge: {
@@ -1672,6 +1685,7 @@ class Tile {
 
   paint() {
     if (!this.route) return;
+    if (this.route !== 'woven-sbs') this.cursor?.idle(); // only a woven pair has depth to measure
     if (this.route === 'woven-sbs') this._paintConv();
     else if (this.route === 'lifted') {
       if (!this.liftLive) paintFlat(this.canvas, this.video); // flat until lift is showing
@@ -1692,13 +1706,16 @@ class Tile {
       c.width = 2 * outW;
       c.height = outH;
     }
-    if (this.call.o.autoConverge) this._sampleDisparity(v, W, H);
+    const hover = !!(this.cursor && this.cursor.hovering);
+    if (this.call.o.autoConverge || hover) this._sampleDisparity(v, W, H, hover);
     const shift = this.conv.step(eyeW);
     const g = c.getContext('2d');
+    const rects = [];
     for (const eye of [0, 1]) {
-      const r = wire.eyeCropRect(eyeW, H, A, shift, eye);
+      const r = (rects[eye] = wire.eyeCropRect(eyeW, H, A, shift, eye));
       g.drawImage(v, eye * eyeW + r.sx, r.sy, r.sw, r.sh, eye * outW, 0, outW, outH);
     }
+    if (this.cursor) this.cursor.draw(g, this.luma, { rL: rects[0], rR: rects[1], outW, outH }, performance.now() / 1000);
   }
 
   /**
@@ -1707,9 +1724,10 @@ class Tile {
    * feedback loop) and hand it to the convergence state, which halves it per eye and low-passes it.
    * A failed measurement holds the last good value.
    */
-  _sampleDisparity(v, W, H) {
+  _sampleDisparity(v, W, H, hover = false) {
     const now = performance.now();
-    if (this.autoPending || now - this.autoAt < AUTO_CONV_INTERVAL_MS) return;
+    // ~15 Hz while a depth cursor hovers: it measures the same copy, under the pointer.
+    if (this.autoPending || now - this.autoAt < (hover ? CURSOR_SAMPLE_MS : AUTO_CONV_INTERVAL_MS)) return;
     this.autoAt = now;
     // Preferred: WebCodecs. `new VideoFrame(video)` + an ASYNC copyTo() of the decoded frame, reading
     // the luma (Y) plane directly: no synchronous GPU->CPU canvas readback on the paint path (that
@@ -1775,6 +1793,8 @@ class Tile {
    */
   _applyMeasurement(img, w, h, s, at, readMs, via) {
     if (this.destroyed) return;
+    if (img) this.luma = { img, w, h, scale: s };
+    if (!this.call.o.autoConverge) return; // the copy was for the depth cursor only
     const t = performance.now();
     const m = img ? this.autoFocus.measure(img, w, h, at) : null;
     const d = this.autoTrack.push(m ? m.d / s : null);
@@ -1800,6 +1820,7 @@ class Tile {
 
   _resetAutoConverge() {
     this.autoGen++;
+    this.luma = null;
     this.autoTrack.reset();
     this.autoFocus.reset();
     this.conv.measuredPx = null;
@@ -1809,6 +1830,8 @@ class Tile {
   destroy() {
     this.destroyed = true;
     this.autoGen++;
+    this.cursor?.dispose();
+    this.cursor = null;
     clearTimeout(this.helloTimer);
     clearTimeout(this.layerTimer);
     clearTimeout(this.leaveTimer);
@@ -1874,6 +1897,7 @@ class SelfTile {
       this.view = addCameraView(wall, this.canvas, cam, {
         mirror: true,
         autoConverge: !!this.call.o.autoConverge, // the call option (default on) governs the self view too
+        cursor: this.call.o.cursor || undefined, // and so does `cursor: 'depth'`
         aspect: this.call.o.tileAspect,
         onRouteChange: (route, st) => {
           if (this.cam !== cam) return; // a view this tile already replaced

package/js/inline3d-camera.js

@@ -39,6 +39,7 @@ import { openCamera as openDevice } from './camera/capture.js';
 import { eyeCropRect, eyeOutputSize, mirrorSwapOps } from './camera/geometry.js';
 import { createConvergence } from './camera/converge.js';
 import { lumaFromRgba, createDisparityTrack, createFocusTracker } from './camera/disparity.js';
+import { resolvePairCursorOption } from './inline3d-cursor-option.js';
 import { buildStereoXmp, jpegWithXmp, webmWithTags, stereoTagEntries, readJpegStereoMeta, readWebmStereoMeta, parseStereoXmp, readJpegXmp } from './camera/metadata.js';
 
 export { readJpegStereoMeta, readWebmStereoMeta, parseStereoXmp, readJpegXmp };
@@ -50,6 +51,8 @@ export const CAMERA_SDK = 'inline3d-camera/1';
 // on a face, a few ms per measurement — the same numbers as the call's remote tiles).
 const AUTO_CONV_INTERVAL_MS = 200;
 const AUTO_CONV_EYE_WIDTH = 240;
+/** While a depth cursor hovers the view, the source is copied this often (it is what the cursor measures). */
+const CURSOR_SAMPLE_MS = 66;
 const FRAME_WAIT_MS = 4000;
 // A self view whose layer the wall could not build (`firstWoven` → `woven:false, 'layer-failed'`)
 // goes flat and re-registers a few times with backoff — the remote tile's rule (web#131).
@@ -521,7 +524,17 @@ class CameraView {
     this._track = createDisparityTrack();
     this._focus = createFocusTracker();
     this._scratch = null;
+    this._luma = null; // the latest grayscale source copy (auto-convergence and the depth cursor read it)
     this.disparityPx = null;
+    // `cursor: 'depth'` (ADR-046 on a pair): validated now, its module loaded only on opt-in.
+    this._cursorOpt = resolvePairCursorOption(o.cursor, '@displayxr/inline3d/camera');
+    /** The depth cursor once its module has loaded; null without `cursor: 'depth'`. */
+    this.cursor = null;
+    if (this._cursorOpt) {
+      import('./camera/pair-cursor.js').then((m) => {
+        if (!this.removed) this.cursor = new m.PairCursor(this.canvas, this._cursorOpt);
+      });
+    }
     this._reroute(false);
     if (this._loop) this._start();
   }
@@ -691,6 +704,7 @@ class CameraView {
   }
 
   _paintFlat(v) {
+    this.cursor?.idle(); // a flat picture has no depth to put a cursor at
     if (!sizeFlat(this.canvas)) return;
     const W = v.videoWidth;
     const H = v.videoHeight;
@@ -726,7 +740,8 @@ class CameraView {
       c.width = 2 * outW;
       c.height = outH;
     }
-    if (this.autoConverge) this._sample(v);
+    const hover = !!(this.cursor && this.cursor.hovering);
+    if (this.autoConverge || hover) this._sample(v, hover);
     const shift = this.conv.step(eyeW);
     const g = c.getContext('2d');
     if (this.mirror) {
@@ -746,15 +761,24 @@ class CameraView {
         g.drawImage(v, eye * eyeW + r.sx, r.sy, r.sw, r.sh, eye * outW, 0, outW, outH);
       }
     }
+    if (this.cursor) {
+      const geo = { rL: eyeCropRect(eyeW, H, A, shift, 0), rR: eyeCropRect(eyeW, H, A, shift, 1), outW, outH, mirror: this.mirror };
+      this.cursor.draw(g, this._luma, geo, performance.now() / 1000);
+    }
   }
 
-  /** Auto-convergence: ~5 Hz, a small grayscale copy of the SOURCE frame (before our shift). */
-  _sample(v) {
+  /**
+   * A small grayscale copy of the SOURCE frame (before our shift): ~5 Hz for auto-convergence,
+   * ~15 Hz while a depth cursor hovers (it measures the same copy under the pointer).
+   */
+  _sample(v, hover = false) {
     const now = performance.now();
-    if (now - this._autoAt < AUTO_CONV_INTERVAL_MS) return;
+    if (now - this._autoAt < (hover ? CURSOR_SAMPLE_MS : AUTO_CONV_INTERVAL_MS)) return;
     this._autoAt = now;
     this._scratch = this._scratch || document.createElement('canvas');
     const g = grabLuma(v, this._scratch);
+    if (g) this._luma = g;
+    if (!this.autoConverge) return;
     const m = g ? this._focus.measure(g.img, g.w, g.h, now) : null;
     const d = this._track.push(m ? m.d / g.scale : null);
     this.conv.measuredPx = d;
@@ -770,6 +794,8 @@ class CameraView {
     clearTimeout(this._layerTimer);
     this._layerTimer = 0;
     this._unregister();
+    this.cursor?.dispose();
+    this.cursor = null;
     this.cam._views?.delete(this);
   }
 }

package/js/inline3d-cursor-option.js

@@ -36,3 +36,21 @@ export function resolveCursorOption(opt, who = '@displayxr/inline3d') {
   }
   return { ...opt };
 }
+
+/** The keys `cursor: { … }` accepts on a stereo PAIR (./camera's addCameraView, ./call's tiles). */
+export const PAIR_CURSOR_OPTION_KEYS = Object.freeze(['height', 'pointerScope']);
+
+/**
+ * `cursor` on a stereo pair — a camera view or a call tile: `'depth'`, or `{ height, pointerScope }`.
+ * A pair has no scene: its depth is MEASURED off the two halves, and the sprite is drawn into them,
+ * so there is no anchor mode (it is always on the line of sight) and the margin is the SDK's. Falsy
+ * is OFF, the zero-cost default.
+ * @returns {null | {height?: number, pointerScope?: string}}
+ */
+export function resolvePairCursorOption(opt, who = '@displayxr/inline3d') {
+  const o = resolveCursorOption(opt, who);
+  if (!o) return null;
+  const unknown = Object.keys(o).filter((k) => !PAIR_CURSOR_OPTION_KEYS.includes(k));
+  if (unknown.length) throw new Error(`${who}: cursor — ${unknown.join(', ')} not supported on a stereo pair; expected ${PAIR_CURSOR_OPTION_KEYS.join(', ')}.`);
+  return o;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@displayxr/inline3d",
-  "version": "1.39.0",
+  "version": "1.40.0",
   "description": "Turn any HTML <canvas> into a 3D window on a spatial display, inside an ordinary web page, for the DisplayXR Browser. Dependency-free; progressive enhancement (falls back to plain 2D everywhere else).",
   "type": "module",
   "types": "./index.d.ts",