@displayxr/inline3d 1.35.0 → 1.37.0

package/js/inline3d-three.js

@@ -52,6 +52,17 @@
 // descriptor and nothing else: no Kooima, no off-axis math, no scale, here or anywhere in this
 // SDK. That stays in the runtime, which is the point of the extension.
 
+import {
+  CursorDepthPlacer,
+  CursorPointer,
+  cursorCrosshairMesh,
+  cursorFootprint,
+  cursorModelMatrix,
+  cursorViewRay,
+  CURSOR_DEFAULT_HEIGHT,
+  CURSOR_DEFAULT_TUNING,
+} from './inline3d-cursor-depth.js';
+
 /**
  * A reusable three.js camera driven directly by an XRView's matrices. Construct once with
  * your THREE namespace and reuse across frames/windows.
@@ -413,3 +424,215 @@ export class EdgeFeather {
     renderer.autoClear = prevAutoClear;
   }
 }
+
+// ── Depth-aware cursor ─────────────────────────────────────────────────────────────────────
+
+/**
+ * A hit test for {@link DepthCursor} that raycasts three.js objects: the first hit along the
+ * ray, as a world point, or null.
+ *
+ * @param {object} THREE  your imported three.js module namespace.
+ * @param {object[]} targets  the Object3Ds that count as content (searched recursively).
+ */
+export function raycastHitTest(THREE, targets) {
+  const raycaster = new THREE.Raycaster();
+  const o = new THREE.Vector3();
+  const d = new THREE.Vector3();
+  return (origin, direction) => {
+    o.set(origin[0], origin[1], origin[2]);
+    d.set(direction[0], direction[1], direction[2]).normalize();
+    raycaster.set(o, d);
+    const hit = raycaster.intersectObjects(targets, true)[0];
+    return hit ? [hit.point.x, hit.point.y, hit.point.z] : null;
+  };
+}
+
+/**
+ * A cursor that rises to the depth of the content under it, so it is never drawn behind
+ * something that pops out of the glass (a depth violation). The page supplies the content —
+ * a hit test — and this does the rest: tracks the pointer over the canvas, hit-tests the
+ * cursor FOOTPRINT (not just the hotspot), places a crosshair sprite with the same maths and
+ * defaults as the runtime's XR_DXR_cursor_depth (ADR-046), and hides the CSS cursor exactly
+ * while the sprite replaces it.
+ *
+ * Opt-in: nothing runs unless you construct one and call update(). In 2D (one view) or with the
+ * pointer off the canvas it is inactive, the sprite is hidden and the CSS cursor is back.
+ *
+ *   const cursor = new DepthCursor(THREE, { canvas, hitTest: raycastHitTest(THREE, [model]) });
+ *   scene.add(cursor.object);
+ *   wall.addScene(canvas, (views, layer) => {
+ *     cursor.update(views);            // first, with THIS frame's views
+ *     // ... render every view as usual; the sprite draws last, depth test off ...
+ *   });
+ *
+ * Head motion: the sprite sits on the line from the viewer through the pointer's point on the
+ * canvas, so its image on the glass does not move when the head does — it stays under the
+ * mouse like the OS cursor, and only its depth follows the content under that line of sight.
+ *
+ * The attach pattern (eye cameras parented under an app camera): pass `viewSpace` = that
+ * parent, so the views' local transforms are carried into world space; keep `cursor.object` at
+ * the scene root.
+ */
+export class DepthCursor {
+  /**
+   * @param {object} THREE  your imported three.js module namespace.
+   * @param {object} opts
+   * @param {HTMLElement} opts.canvas  the woven canvas (pointer is tracked over it).
+   * @param {(origin:number[], direction:number[]) => number[]|null} opts.hitTest  nearest
+   *        content point along a ray in WORLD space, or null. {@link raycastHitTest} for meshes;
+   *        for splats return the renderer's expected depth along the ray (never raycast splats).
+   * @param {number} [opts.height=0.03]  sprite height as a fraction of the canvas height.
+   * @param {number} [opts.color=0xffd61a]  fill colour (the outline is always dark).
+   * @param {number} [opts.margin=0.005]  how far in front of the content it floats, in eye-baseline
+   *        units (0.005 ≈ 1.5 mm at 60 cm: it rests on the content).
+   * @param {'hybrid'|'screen'|'world'} [opts.anchor='hybrid']  where along that depth: 'hybrid'
+   *        follows the pointer exactly while it moves and stays world-fixed (parallaxes with the
+   *        content) while it is still; 'screen' never parallaxes; 'world' always does.
+   * @param {'canvas'|'window'} [opts.pointerScope='canvas']  'window' keeps the cursor over DOM
+   *        layered on the canvas (overlay buttons) and hides the CSS cursor page-wide meanwhile.
+   * @param {object} [opts.viewSpace]  Object3D whose world matrix maps view transforms to world.
+   * @param {number} [opts.raysPerFrame=0]  0 = the whole footprint every frame (two eyes × 9
+   *        points). N > 0 = an EXPENSIVE hit test (e.g. a gaussian-splat raycast, ~8 ms a ray):
+   *        cast N rays a frame from the first eye, cycling through the footprint, and take the
+   *        nearest of the most recent full cycle.
+   */
+  constructor(
+    THREE,
+    { canvas, hitTest, height = CURSOR_DEFAULT_HEIGHT, color = 0xffd61a, viewSpace = null, raysPerFrame = 0, margin, anchor = 'hybrid', pointerScope = 'canvas' },
+  ) {
+    this._THREE = THREE;
+    this.canvas = canvas;
+    this.hitTest = hitTest;
+    this.height = height;
+    this.viewSpace = viewSpace;
+    this.raysPerFrame = raysPerFrame;
+    this._ring = []; // amortised mode: the last hit per footprint point (null = a miss)
+    this._next = 0;
+    const tuning = margin > 0 && Number.isFinite(margin) ? { ...CURSOR_DEFAULT_TUNING, margin } : CURSOR_DEFAULT_TUNING;
+    this.placer = new CursorDepthPlacer(tuning, { anchor });
+    this.pointer = new CursorPointer(canvas, { scope: pointerScope });
+    /** The last placement (diagnostics): `{active, position, height, disparity, targetDisparity, anchored}`. */
+    this.placement = { active: false };
+
+    // Filled strokes with a dark outline (a 1 px line reads too thin through the lens).
+    const fill = [((color >> 16) & 255) / 255, ((color >> 8) & 255) / 255, (color & 255) / 255, 1];
+    const m = cursorCrosshairMesh(fill);
+    const geom = new THREE.BufferGeometry();
+    geom.setAttribute('position', new THREE.Float32BufferAttribute(m.positions, 3));
+    geom.setAttribute('color', new THREE.Float32BufferAttribute(m.colors, 4));
+    const mat = new THREE.MeshBasicMaterial({
+      vertexColors: true,
+      transparent: true,
+      depthTest: false,
+      depthWrite: false,
+      side: THREE.DoubleSide,
+      toneMapped: false,
+    });
+    this.object = new THREE.Mesh(geom, mat);
+    this.object.name = 'inline3d-cursor';
+    this.object.matrixAutoUpdate = false;
+    this.object.frustumCulled = false;
+    this.object.renderOrder = 1e9; // last: never occluded, it is in front by construction
+    this.object.visible = false;
+  }
+
+  /** Set the pointer position directly (canvas-normalised, v down), or null — for scripted input. */
+  setPointer(u, v) {
+    this.pointer.set(u, v);
+  }
+
+  /**
+   * Place the sprite for this frame. Call with the views you are about to render.
+   * @param {ArrayLike<XRView|{projectionMatrix:ArrayLike<number>,transformMatrix:ArrayLike<number>}>} views
+   * @param {number} [nowMs=performance.now()]
+   */
+  update(views, nowMs = globalThis.performance ? globalThis.performance.now() : Date.now()) {
+    const vs = this._worldViews(views);
+    const uv = this.pointer.uv;
+    let p = { active: false };
+    if (uv && vs.length >= 2) {
+      const [u, v] = uv;
+      const nearestPoint = this._footprintHit(vs, u, v);
+      p = this.placer.update(vs, { u, v, nearestPoint, cursorHeight: this.height }, nowMs / 1000);
+    }
+    this.placement = p;
+    this.object.visible = !!p.active;
+    if (p.active) {
+      this.object.matrix.fromArray(cursorModelMatrix(p));
+      this.object.matrixWorldNeedsUpdate = true;
+    }
+    this.pointer.hideCss(!!p.active);
+    return p;
+  }
+
+  /** Remove listeners, give the CSS cursor back, free the sprite's GPU resources. */
+  dispose() {
+    this.pointer.dispose();
+    this.object.visible = false;
+    this.object.geometry.dispose();
+    this.object.material.dispose();
+  }
+
+  // Nearest content point under the footprint, from the two outer views (or, amortised, from
+  // the first eye over the last cycle of footprint points).
+  _footprintHit(views, u, v) {
+    const a = views[0], b = views[views.length - 1];
+    const m = a.transformMatrix;
+    const fl = Math.hypot(m[8], m[9], m[10]) || 1;
+    const f = [-m[8] / fl, -m[9] / fl, -m[10] / fl];
+    const fp = cursorFootprint(u, v, this.height, this.pointer.aspect());
+    let candidates;
+    if (this.raysPerFrame > 0) {
+      const ring = this._ring;
+      ring.length = fp.length;
+      for (let i = 0; i < this.raysPerFrame; i++) {
+        const k = this._next++ % fp.length;
+        const ray = cursorViewRay(a, fp[k][0], fp[k][1]);
+        ring[k] = this.hitTest(ray.origin, ray.direction);
+      }
+      candidates = ring;
+    } else {
+      candidates = [];
+      for (const [su, sv] of fp) {
+        for (const view of [a, b]) {
+          const ray = cursorViewRay(view, su, sv);
+          candidates.push(this.hitTest(ray.origin, ray.direction));
+        }
+      }
+    }
+    let best = Infinity, nearest = null;
+    for (const p of candidates) {
+      if (!p) continue;
+      const depth = p[0] * f[0] + p[1] * f[1] + p[2] * f[2]; // smaller = nearer the viewer
+      if (depth < best) {
+        best = depth;
+        nearest = p;
+      }
+    }
+    return nearest;
+  }
+
+  _worldViews(views) {
+    const out = [];
+    if (!views) return out;
+    const parent = this.viewSpace ? this.viewSpace.matrixWorld.elements : null;
+    for (const v of views) {
+      const t = v.transformMatrix || v.transform.matrix;
+      out.push({ projectionMatrix: v.projectionMatrix, transformMatrix: parent ? mul4(parent, t) : t });
+    }
+    return out;
+  }
+}
+
+/** Column-major 4x4 multiply, a × b. */
+function mul4(a, b) {
+  const o = new Array(16);
+  for (let c = 0; c < 4; c++) {
+    for (let r = 0; r < 4; r++) {
+      let s = 0;
+      for (let k = 0; k < 4; k++) s += a[k * 4 + r] * b[c * 4 + k];
+      o[c * 4 + r] = s;
+    }
+  }
+  return o;
+}

package/js/inline3d-viewer.js

@@ -737,6 +737,11 @@ export class SceneViewer {
     this._cacheGood(views, vps, !eye);
     const g = this._lastGood;
     if (eye) (this.viewerEase ||= viewerEaseFor(frame, this._viewerEaseOpt)).apply(g.entries, frameTrackingState(frame));
+    // cursor: 'depth' (ADR-046) — only when a page opted in; null otherwise.
+    if (this.cursorDepth) {
+      this.scene.updateMatrixWorld(); // the hit test reads THIS frame's subject pose
+      this.cursorDepth.update(g.entries.map((e) => ({ projectionMatrix: e.proj, transformMatrix: e.pose })));
+    }
     const r = this.renderer;
     // getViewport() splits canvas.width; a drawing buffer the browser clamped behind our back
     // (bufferScale().mismatch) gets the same split mapped onto its real size, never canvas.width/2.
@@ -771,6 +776,22 @@ export class SceneViewer {
     return this;
   }
 
+  /**
+   * `cursor: 'depth'` (ADR-046): a cursor that rises to the subject under it instead of being
+   * drawn on the glass behind content that pops out. Called by ./model and ./splat on opt-in
+   * only — without it nothing is built and every hook below is skipped. Injected, like
+   * useEyeCamera, so this module never imports three.js itself.
+   * @param {Function} DepthCursorClass  ./three's DepthCursor.
+   * @param {(content: object) => Function} hitTestFor  builds the hit test over the subject group.
+   * @param {object} [opts]  extra DepthCursor options (./splat passes `raysPerFrame`).
+   */
+  useDepthCursor(DepthCursorClass, hitTestFor, opts = {}) {
+    if (this.cursorDepth) return this;
+    this.cursorDepth = new DepthCursorClass(this._THREE, { ...opts, canvas: this.canvas, hitTest: hitTestFor(this.content) });
+    this.scene.add(this.cursorDepth.object);
+    return this;
+  }
+
   /** Drive a flat, single-camera render loop for browsers without inline-3D. */
   startMono() {
     if (this._monoRaf || this._disposed) return;
@@ -780,6 +801,7 @@ export class SceneViewer {
       if (this._disposed) return;
       this._monoRaf = requestAnimationFrame(loop);
       this._tick();
+      this.cursorDepth?.update(null); // 2D: no sprite, the normal cursor
       const r = this.renderer;
       r.clear();
       const b = this._bufScale();
@@ -809,6 +831,8 @@ export class SceneViewer {
     if (this._ro) this._ro.disconnect();
     else removeEventListener('resize', this._onResize);
     this._unbindOrbit();
+    this.cursorDepth?.dispose();
+    this.cursorDepth = null;
     this.renderer.dispose();
   }
 

package/model.d.ts

@@ -1,6 +1,7 @@
 // Type definitions for @displayxr/inline3d/model (and ./model/three).
 // EXPERIMENTAL — not covered by the 1.x semver promise. See docs/sdk-stability.md.
 
+import type { CursorOptions } from './cursor-depth.js';
 import type { SceneViewer, SubjectBounds, OrbitPose } from './viewer.js';
 import type { FirstWovenResult, ViewerEaseOption } from './index.js';
 import type { PlayCanvasSplatViewer, SplatCameraPose, SplatFrameInfo } from './splat.js';
@@ -48,6 +49,17 @@ export interface ModelOptions {
    * `createInline3D({ viewerEase })`, else on. `false` restores the snap.
    */
   viewerEase?: ViewerEaseOption;
+  /**
+   * `'depth'`: a cursor that rises to the subject under it instead of being drawn on the glass,
+   * behind content that pops out (ADR-046, same maths as native `XR_DXR_cursor_depth`). A
+   * crosshair just in front of the nearest content under the pointer's footprint; the CSS cursor
+   * is hidden while it shows and comes back in 2D and off the canvas. OPT-IN: without it nothing
+   * is loaded or run. PlayCanvas: hit-tested with the engine's depth picker (meshes and splats,
+   * one low-res pass every other frame while the pointer is over the tile). three: a raycast
+   * against the subject. Pass an object to tune it: `{ margin, anchor, height, pointerScope }`
+   * (any object also means depth).
+   */
+  cursor?: 'depth' | CursorOptions;
   /**
    * How the mesh is lit.
    *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@displayxr/inline3d",
-  "version": "1.35.0",
+  "version": "1.37.0",
   "description": "Turn any HTML <canvas> into a 3D window on a spatial display, inside an ordinary web page, for the DisplayXR Browser. Dependency-free; progressive enhancement (falls back to plain 2D everywhere else).",
   "type": "module",
   "types": "./index.d.ts",
@@ -15,6 +15,10 @@
       "types": "./three.d.ts",
       "import": "./js/inline3d-three.js"
     },
+    "./cursor-depth": {
+      "types": "./cursor-depth.d.ts",
+      "import": "./js/inline3d-cursor-depth.js"
+    },
     "./viewer": {
       "types": "./viewer.d.ts",
       "import": "./js/inline3d-viewer.js"
@@ -70,6 +74,9 @@
     "js/inline3d-buffer-limit.js",
     "js/inline3d-viewer-ease.js",
     "js/inline3d-three.js",
+    "js/inline3d-cursor-option.js",
+    "js/inline3d-cursor-depth.js",
+    "js/inline3d-cursor-depth-playcanvas.js",
     "js/inline3d-viewer.js",
     "js/inline3d-splat.js",
     "js/inline3d-splat-pc-entry.js",
@@ -100,6 +107,7 @@
     "dist/",
     "index.d.ts",
     "three.d.ts",
+    "cursor-depth.d.ts",
     "viewer.d.ts",
     "splat.d.ts",
     "splat-playcanvas.d.ts",
@@ -167,6 +175,7 @@
     "prepack": "node tools/dist/build.mjs",
     "test:e2e": "node test/e2e/call-element.e2e.mjs",
     "test:e2e:camera": "node test/e2e/camera.e2e.mjs",
+    "test:e2e:cursor": "node test/e2e/cursor-depth.e2e.mjs",
     "api:check": "node tools/api-surface.mjs --check",
     "api:update": "node tools/api-surface.mjs --update",
     "check:snippets": "node tools/check-snippets.mjs",

package/splat.d.ts

@@ -1,6 +1,7 @@
 // Type definitions for @displayxr/inline3d/splat.
 // EXPERIMENTAL — not covered by the 1.x semver promise. See docs/sdk-stability.md.
 
+import type { CursorOptions } from './cursor-depth.js';
 import type { SceneViewer, SubjectBounds, OrbitPose } from './viewer.js';
 import type { FirstWovenResult, ViewerEaseOption, DisplayMetrics } from './index.js';
 
@@ -472,6 +473,17 @@ export interface SplatOptions {
    * `createInline3D({ viewerEase })`, else on. `false` restores the snap.
    */
   viewerEase?: ViewerEaseOption;
+  /**
+   * `'depth'`: a cursor that rises to the subject under it instead of being drawn on the glass,
+   * behind content that pops out (ADR-046, same maths as native `XR_DXR_cursor_depth`). A
+   * crosshair just in front of the nearest content under the pointer's footprint; the CSS cursor
+   * is hidden while it shows and comes back in 2D and off the canvas. OPT-IN: without it nothing
+   * is loaded or run. PlayCanvas: hit-tested with the engine's depth picker (meshes and splats,
+   * one low-res pass every other frame while the pointer is over the tile). three: a raycast
+   * against the subject. Pass an object to tune it: `{ margin, anchor, height, pointerScope }`
+   * (any object also means depth).
+   */
+  cursor?: 'depth' | CursorOptions;
   /** Camera rig only: the distance in world metres that sits ON the glass. */
   convergence?: number;
   /**

package/three.d.ts

@@ -142,3 +142,60 @@ export class EdgeFeather {
     vp: { x: number; y: number; width: number; height: number },
   ): void;
 }
+
+import type { CursorAnchorMode, CursorPlacement, CursorView } from './cursor-depth.js';
+
+/**
+ * A hit test for {@link DepthCursor} that raycasts three.js objects (recursively): the first hit
+ * along the ray as a world point, or null.
+ */
+export function raycastHitTest(
+  THREE: unknown,
+  targets: unknown[],
+): (origin: number[], direction: number[]) => number[] | null;
+
+/**
+ * A cursor that rises to the depth of the content under it, so it is never drawn behind something
+ * that pops out of the glass. Same maths and defaults as the runtime's XR_DXR_cursor_depth
+ * (ADR-046). Tracks the pointer over `canvas`, hit-tests the cursor FOOTPRINT, places a crosshair
+ * (`object`, add it to your scene) and hides the CSS cursor exactly while the crosshair replaces it.
+ * Opt-in: nothing runs unless you construct one and call `update(views)` each frame.
+ *
+ * Head motion leaves the crosshair's image on the glass where it is (it stays under the mouse like
+ * the OS cursor); only its depth follows the content under that line of sight.
+ */
+export class DepthCursor {
+  constructor(
+    THREE: unknown,
+    opts: {
+      /** The woven canvas; the pointer is tracked over it. */
+      canvas: HTMLElement;
+      /** Nearest content point along a WORLD ray, or null. For splats return the expected depth along the ray. */
+      hitTest: (origin: number[], direction: number[]) => number[] | null;
+      /** Sprite height as a fraction of the canvas height (default 0.03). */
+      height?: number;
+      /** Fill colour of the crosshair (default 0xffd61a; the outline is always dark). */
+      color?: number;
+      /** How far in front of the content it floats, in eye-baseline units (default 0.005 ≈ 1.5 mm at 60 cm). */
+      margin?: number;
+      /** Default `'hybrid'`: exact while the pointer moves, world-fixed (parallaxing) while it is still. */
+      anchor?: CursorAnchorMode;
+      /** `'window'`: keep the cursor over DOM layered on the canvas; hides the CSS cursor page-wide meanwhile. */
+      pointerScope?: 'canvas' | 'window';
+      /** An EXPENSIVE hit test: N rays a frame from the first eye, cycling the footprint (0 = all). */
+      raysPerFrame?: number;
+      /** Attach pattern: the Object3D the eye cameras hang off; its world matrix carries the views to world. */
+      viewSpace?: unknown;
+    },
+  );
+  /** The crosshair (a `THREE.Mesh` of filled, outlined strokes; depth test off, drawn last). Add it to the scene root. */
+  readonly object: unknown;
+  /** The last placement, for diagnostics. */
+  readonly placement: CursorPlacement | { active: false };
+  /** Place the sprite for this frame — call with the views you are about to render, before rendering. */
+  update(views: ArrayLike<XRView | CursorView>, nowMs?: number): CursorPlacement | { active: false };
+  /** Set the pointer directly (canvas-normalised, v down), or null — for scripted input. */
+  setPointer(u: number | null, v?: number): void;
+  /** Remove listeners, restore the CSS cursor, free the sprite's GPU resources. */
+  dispose(): void;
+}