@displayxr/inline3d 1.34.0 → 1.36.0

package/js/inline3d-call.js

@@ -1127,6 +1127,9 @@ class Call {
       hello: this.hello(),
       transport: this.transport,
       peers: [...this.tiles.values()].map((tile) => tile.diag()),
+      // The self view's ACTUAL weave state (web#131): route, woven, the fallback reason and the
+      // tile's firstWoven result — null without a self view.
+      self: this.self ? this.self.diag() : null,
       mono3D: { ...this._mono3DInfo(), lift: this.liftApi, pool: this.liftPool ? this.liftPool.list() : [] },
       speaker: this.speakerId,
       depth: this.depth,
@@ -1866,14 +1869,34 @@ class SelfTile {
     else {
       this.detach();
       this.cam = cam;
-      this.view = addCameraView(wall, this.canvas, cam, { mirror: true, autoConverge: false, aspect: this.call.o.tileAspect });
+      // onRouteChange: the view drops to its flat left eye by itself when the wall says the tile
+      // will not weave (firstWoven → woven:false, web#131), and the badge must follow it.
+      this.view = addCameraView(wall, this.canvas, cam, {
+        mirror: true,
+        autoConverge: false,
+        aspect: this.call.o.tileAspect,
+        onRouteChange: (route, st) => {
+          if (this.cam !== cam) return; // a view this tile already replaced
+          this.call.log('self-route', { route, woven: st.woven, reason: st.reason });
+          this._badge();
+        },
+      });
     }
     this._badge();
   }
 
+  /** 3D only while the view is on the woven route — a layer the wall failed has already taken it flat. */
   _badge() {
     const t = this.call.t;
-    this.badge.replaceChildren(el('b', { text: `${t('you')} · ${this.route === 'woven-sbs' ? t('badge3D') : t('badge2D')}` }));
+    const woven = !!(this.view && this.view.woven);
+    this.badge.replaceChildren(el('b', { text: `${t('you')} · ${woven ? t('badge3D') : t('badge2D')}` }));
+    this.badge.dataset.route = this.route || '';
+  }
+
+  /** For `call.diagnostics().self`: what the self view actually shows. */
+  diag() {
+    if (!this.view) return { route: null, woven: false, reason: 'no-camera', firstWoven: null, layerRetries: 0 };
+    return this.view.weaveState();
   }
 
   onWallLost() {

package/js/inline3d-camera.js

@@ -51,6 +51,11 @@ export const CAMERA_SDK = 'inline3d-camera/1';
 const AUTO_CONV_INTERVAL_MS = 200;
 const AUTO_CONV_EYE_WIDTH = 240;
 const FRAME_WAIT_MS = 4000;
+// A self view whose layer the wall could not build (`firstWoven` → `woven:false, 'layer-failed'`)
+// goes flat and re-registers a few times with backoff — the remote tile's rule (web#131).
+const LAYER_RETRIES = 4;
+const LAYER_RETRY_BASE_MS = 1500;
+const LAYER_RETRY_MAX_MS = 20000;
 const RECORD_MIME_CANDIDATES = ['video/webm;codecs=vp9', 'video/webm;codecs=vp8', 'video/webm', 'video/mp4'];
 
 const hasDoc = () => typeof document !== 'undefined' && document && typeof document.createElement === 'function';
@@ -471,7 +476,12 @@ function sizeFlat(canvas) {
  * @param {object|null} wall  a `createInline3D()` / `sharedInline3D()` result (may be unsupported) or null
  * @param {HTMLCanvasElement} canvas  the tile — its CSS box is the shape the viewer sees
  * @param {StereoCamera} cam
- * @param {{ mirror?: boolean, autoConverge?: boolean, depth?: number, aspect?: number }} [opts]
+ * On a 3D wall the view follows its tile's `firstWoven`: a window the wall will not weave
+ * (`'layer-failed'`, `'session-ended'`) drops the view to the flat left eye — never the packed
+ * pair — with `fallbackReason` saying why, and `onRouteChange(route, state)` tells the page (a
+ * badge reads `route`). A refused layer is retried a few times; `_reroute(true, wall)` recovers.
+ *
+ * @param {{ mirror?: boolean, autoConverge?: boolean, depth?: number, aspect?: number, onRouteChange?: Function }} [opts]
  */
 export function addCameraView(wall, canvas, cam, opts = {}) {
   if (!canvas || typeof canvas.getContext !== 'function') throw new TypeError(`${TAG} addCameraView(wall, canvas, cam, opts) needs a canvas`);
@@ -494,6 +504,15 @@ class CameraView {
     this.setDepth(o.depth);
     this.route = null;
     this.handle = null;
+    // Why a pair on a 3D-capable wall is shown FLAT (web#131): the wall said this tile's window
+    // will not weave (`firstWoven` → woven:false — 'layer-failed', 'session-ended'). Null while
+    // nothing has failed. A packed pair is never the visible fallback: the view paints one eye.
+    this.fallbackReason = null;
+    // The current registration's settled `firstWoven`, null while pending / not registered.
+    this._fw = null;
+    this._onRouteChange = typeof o.onRouteChange === 'function' ? o.onRouteChange : null;
+    this._layerFails = 0;
+    this._layerTimer = 0;
     this.buffer = null; // the SBS canvas the wall repaints the tile from (woven route)
     this.removed = false;
     this._raf = 0;
@@ -507,11 +526,30 @@ class CameraView {
     if (this._loop) this._start();
   }
 
-  /** `'woven-sbs'` (3D on the panel) | `'flat-left'` (a pair, shown flat) | `'flat'` (mono). */
+  /**
+   * True while the view is on the woven route: registered on a 3D wall, and the wall has not said
+   * the tile will not weave (a failed layer drops the view to `'flat-left'`). Route is
+   * `'woven-sbs'` (3D on the panel) | `'flat-left'` (a pair, shown flat) | `'flat'` (mono).
+   */
   get woven() {
     return this.route === 'woven-sbs';
   }
 
+  /**
+   * What is actually on the panel, for diagnostics: `{ route, woven, reason, firstWoven,
+   * layerRetries }`. `firstWoven` is the current registration's settled result, `'pending'`
+   * before it settles, null off the woven route.
+   */
+  weaveState() {
+    return {
+      route: this.route,
+      woven: this.woven,
+      reason: this.fallbackReason,
+      firstWoven: this.route === 'woven-sbs' ? (this._fw ? { ...this._fw } : 'pending') : null,
+      layerRetries: this._layerFails,
+    };
+  }
+
   /** The per-eye convergence shift currently painted, source px. */
   get convergencePx() {
     return this.conv.current;
@@ -543,20 +581,77 @@ class CameraView {
     return this.autoConverge;
   }
 
-  /** Re-register on the wall (the call does this when the weave goes live, #172, or a session is recovered). */
+  /**
+   * Re-register on the wall (the call does this when the weave goes live, #172, or a session is
+   * recovered). A FORCED reroute also clears a layer fallback (web#131): a wall that came back
+   * gets a fresh registration. An unforced one keeps the view flat while a fallback stands.
+   */
   _reroute(force, wall) {
     if (wall !== undefined) this.wall = wall;
     if (this.removed) return;
+    if (force && this.fallbackReason) {
+      this.fallbackReason = null;
+      clearTimeout(this._layerTimer);
+      this._layerTimer = 0;
+    }
     const cam = this.cam;
     const live = cam.state === 'live' && cam.video;
-    const route = cam.format === 'sbs' && live && this.wall && this.wall.supported ? 'woven-sbs' : cam.format === 'sbs' ? 'flat-left' : 'flat';
+    const wovenOk = cam.format === 'sbs' && live && this.wall && this.wall.supported && !this.fallbackReason;
+    const route = wovenOk ? 'woven-sbs' : cam.format === 'sbs' ? 'flat-left' : 'flat';
     if (!force && route === this.route) return;
+    const prev = this.route;
     this._unregister();
     this.route = route;
-    if (route === 'woven-sbs') {
-      if (!this.buffer) this.buffer = document.createElement('canvas');
-      this._paintWoven(); // the first frame exists before the layer does
-      this.handle = this.wall.addImage(this.canvas, this.buffer);
+    if (route === 'woven-sbs') this._registerWoven();
+    if (route !== prev) this._routeChanged();
+  }
+
+  _registerWoven() {
+    if (!this.buffer) this.buffer = document.createElement('canvas');
+    this._paintWoven(); // the first frame exists before the layer does
+    const handle = this.wall.addImage(this.canvas, this.buffer);
+    this.handle = handle;
+    this._fw = null;
+    const fw = handle && handle.firstWoven;
+    if (!fw || typeof fw.then !== 'function') return;
+    fw.then((res) => {
+      // A result for a registration we already left (removed, re-registered) says nothing now.
+      if (this.handle !== handle || this.removed || !res) return;
+      this._fw = { woven: !!res.woven, confirmed: !!res.confirmed, reason: res.reason || null, ms: res.ms };
+      this.cam.log?.('view-first-woven', { woven: !!res.woven, reason: res.reason || null });
+      if (res.woven) {
+        this._layerFails = 0;
+        this._routeChanged(); // the state settled; the route did not move
+      } else if (res.reason !== 'removed') this._layerLost(res.reason || 'layer-failed');
+    });
+  }
+
+  /**
+   * The wall said this window will not weave (web#131): the canvas must never be left showing the
+   * packed pair under a 3D badge. Go flat (one eye, mirrored) and say why. A layer the browser
+   * refused is retried a few times with backoff while the wall stays up (the remote tile's rule);
+   * a session that ended waits for `_reroute(true, wall)` with the new wall.
+   */
+  _layerLost(reason) {
+    this.fallbackReason = reason;
+    this._reroute(false);
+    if (reason !== 'layer-failed' || this._layerFails >= LAYER_RETRIES) return;
+    const delay = Math.min(LAYER_RETRY_MAX_MS, LAYER_RETRY_BASE_MS * Math.pow(2, this._layerFails++));
+    this.cam.log?.('view-layer-retry', { attempt: this._layerFails, inMs: delay });
+    clearTimeout(this._layerTimer);
+    this._layerTimer = setTimeout(() => {
+      this._layerTimer = 0;
+      if (this.removed || this.fallbackReason !== reason) return;
+      this._reroute(true);
+    }, delay);
+  }
+
+  _routeChanged() {
+    if (!this._onRouteChange) return;
+    try {
+      this._onRouteChange(this.route, this.weaveState());
+    } catch (err) {
+      console.error(`${TAG} onRouteChange threw`, err);
     }
   }
 
@@ -672,6 +767,8 @@ class CameraView {
     this.removed = true;
     if (this._raf) cancelAnimationFrame(this._raf);
     this._raf = 0;
+    clearTimeout(this._layerTimer);
+    this._layerTimer = 0;
     this._unregister();
     this.cam._views?.delete(this);
   }

package/js/inline3d-cursor-depth-playcanvas.js

@@ -0,0 +1,281 @@
+// inline3d-cursor-depth-playcanvas.js — the depth-aware cursor on the PlayCanvas backend (./model and
+// ./splat, `cursor: 'depth'`).
+//
+// Internal, and LOADED ONLY ON OPT-IN: the adapters import this module dynamically when a page
+// passes `cursor: 'depth'`, so a page that does not ask pays nothing — not even the download.
+// The adapter's per-frame hook is an optional call (`viewer.cursorDepth?.frame(…)`) that is a
+// no-op without it.
+//
+// Hit test: PlayCanvas's own depth PICKER, from the first eye, at low resolution. One path for
+// meshes AND gaussian splats (the picker renders both into its depth target), and it reads what
+// is actually drawn — no splat raycasting, no CPU triangle walk. It costs one extra low-res
+// scene pass, run every other frame and only while the pointer is over the canvas. Results come
+// back asynchronously (a frame or two late); the placement filter absorbs that, exactly like the
+// runtime's one-frame-late app hint.
+//
+// Placement: the same maths as everywhere else (./inline3d-cursor-depth.js, a port of the
+// runtime's u_cursor_depth, ADR-046), solved in WORLD space from this frame's views carried
+// through the rig node. Drawn as immediate-mode lines with depth test off, so every eye camera
+// draws it and nothing occludes it.
+
+import {
+  CursorDepthPlacer,
+  CursorPointer,
+  CURSOR_DEFAULT_HEIGHT,
+  cursorCrosshairPoints,
+  cursorFootprint,
+  cursorViewRay,
+} from './inline3d-cursor-depth.js';
+
+/** Pick-buffer width in pixels (height follows the eye's aspect). */
+const PICK_WIDTH = 160;
+/** Pick every Nth frame. */
+const PICK_EVERY = 2;
+/** A pick that has not answered in this long is abandoned (and the cursor falls to the glass). */
+const PICK_TIMEOUT_MS = 500;
+
+export class PlayCanvasDepthCursor {
+  /**
+   * @param {object} pc  the PlayCanvas namespace the adapter renders with.
+   * @param {object} viewer  the PlayCanvasSplatViewer (app, rigNode, eye are read lazily).
+   * @param {{canvas: HTMLElement, height?: number, color?: number[]}} opts
+   */
+  constructor(pc, viewer, { canvas, height = CURSOR_DEFAULT_HEIGHT, color = [1, 0.83, 0.1, 1] }) {
+    this.pc = pc;
+    this.viewer = viewer;
+    this.height = height;
+    this.color = color;
+    this.pointer = new CursorPointer(canvas);
+    this.placer = new CursorDepthPlacer();
+    /** The last placement (diagnostics). */
+    this.placement = { active: false };
+    /** The last picked world point under the footprint, or null (nothing under it). */
+    this.hit = null;
+    this._picker = null;
+    this._pickCam = null;
+    this._inFlight = false;
+    this._pickId = 0;
+    this._pickStart = 0;
+    this._frame = 0;
+    this._disposed = false;
+    this._points = [];
+    this._colors = [];
+  }
+
+  /**
+   * One 3D frame, before app.tick: schedule a pick, place the sprite, queue its lines.
+   * @param {Array<{proj: ArrayLike<number>, pose: ArrayLike<number>, width: number, height: number}>} entries
+   *        this frame's views, poses in RIG space (the eye cameras' parent).
+   */
+  frame(entries) {
+    const app = this.viewer.app;
+    const rig = this.viewer.rigNode;
+    const uv = this.pointer.uv;
+    if (this._disposed || !app || !rig || !uv || !entries || entries.length < 2) {
+      this.inactive();
+      return;
+    }
+    const R = rig.getWorldTransform().data;
+    const views = entries.map((e) => ({ projectionMatrix: e.proj, transformMatrix: mul4(R, e.pose) }));
+    const t = globalThis.performance ? globalThis.performance.now() : Date.now();
+    if (this._inFlight && t - this._pickStart > PICK_TIMEOUT_MS) {
+      // A read that never settles must not freeze the cursor on a stale hit.
+      this._inFlight = false;
+      this._pickId++;
+      this.hit = null;
+      if (!this._warnedTimeout) {
+        this._warnedTimeout = true;
+        console.warn('[inline3d] cursor: depth: a PlayCanvas pick did not answer in time; retrying (cursor on the glass meanwhile).');
+      }
+    }
+    if (this._frame++ % PICK_EVERY === 0 && !this._inFlight) this._pick(app, rig, entries[0], uv);
+    const now = (globalThis.performance ? globalThis.performance.now() : Date.now()) / 1000;
+    const p = this.placer.update(views, { u: uv[0], v: uv[1], nearestPoint: this.hit, cursorHeight: this.height }, now);
+    this.placement = p;
+    this.pointer.hideCss(!!p.active);
+    if (!p.active) return;
+    const pts = cursorCrosshairPoints(p, this._points);
+    const cols = this._colors;
+    if (cols.length !== (pts.length / 3) * 4) {
+      cols.length = 0;
+      for (let i = 0; i < pts.length / 3; i++) cols.push(...this.color);
+    }
+    app.drawLineArrays(pts, cols, false);
+  }
+
+  /** Mono / no pointer / no views: no sprite, the CSS cursor back. */
+  inactive() {
+    if (this.placement.active) this.placement = { active: false };
+    this.pointer.hideCss(false);
+  }
+
+  dispose() {
+    this._disposed = true;
+    this.pointer.dispose();
+    try {
+      this._picker?.destroy?.();
+      this._pickCam?.destroy?.();
+    } catch {
+      // the app may already be gone with its device
+    }
+    this._picker = null;
+    this._pickCam = null;
+  }
+
+  // Render the pick buffer from the first eye and read the footprint's world points.
+  _pick(app, rig, e, uv) {
+    const pc = this.pc;
+    const aspect = e.width > 0 && e.height > 0 ? e.width / e.height : 1;
+    const W = PICK_WIDTH;
+    const H = Math.max(1, Math.round(W / aspect));
+    if (!this._picker) {
+      this._picker = new pc.Picker(app, W, H, true);
+      const cam = new pc.Entity('inline3d-cursor-pick', app);
+      // frustumCulling off: this camera never renders on its own (disabled — the picker drives
+      // it), so nothing keeps its culling frustum current, and a stale one culls everything.
+      cam.addComponent('camera', { clearColor: new pc.Color(0, 0, 0, 0), frustumCulling: false });
+      cam.camera.enabled = false;
+      cam._dxrProj = new Float64Array(16);
+      cam.camera.calculateProjection = (out) => out.set(cam._dxrProj);
+      rig.addChild(cam);
+      this._pickCam = cam;
+    }
+    const cam = this._pickCam;
+    cam._dxrProj.set(e.proj);
+    // The picker linearises depth with the COMPONENT's near/far, and reconstructs through the
+    // projection: keep the two in agreement (GL-style projection: m10 = -(f+n)/(f-n), m14 = -2fn/(f-n)).
+    const m10 = e.proj[10], m14 = e.proj[14];
+    const near = m14 / (m10 - 1);
+    const far = m14 / (m10 + 1);
+    if (near > 0 && far > near) {
+      cam.camera.nearClip = near;
+      cam.camera.farClip = far;
+    }
+    if (cam.camera.camera) cam.camera.camera._projMatDirty = true;
+    const q = quatFromMatrix(e.pose);
+    cam.setLocalPosition(e.pose[12], e.pose[13], e.pose[14]);
+    cam.setLocalRotation(q[0], q[1], q[2], q[3]);
+    const eye = this.viewer.eye?.camera;
+    const skip = new Set([pc.LAYERID_UI, pc.LAYERID_IMMEDIATE, pc.LAYERID_SKYBOX]);
+    const ids = (eye && Array.isArray(eye.layers) ? eye.layers : cam.camera.layers).filter((id) => !skip.has(id));
+    const layers = ids.map((id) => app.scene.layers.getLayerById(id)).filter(Boolean);
+
+    const picker = this._picker;
+    picker.resize(W, H);
+    try {
+      // Enabled only for the synchronous pick pass: a disabled camera is not in its layers'
+      // camera lists, so the picker would render nothing; enabled outside it, the camera would
+      // render a frame of its own on every app.tick.
+      cam.camera.enabled = true;
+      picker.prepare(cam.camera, app.scene, layers);
+    } catch (err) {
+      console.warn('[inline3d] cursor: depth: the PlayCanvas picker failed; cursor stays on the glass.', err);
+      this.hit = null;
+      return;
+    } finally {
+      cam.camera.enabled = false;
+    }
+    // The picker stores LINEAR depth: (viewDist - near) / (far - near), viewDist = 1/gl_FragCoord.w.
+    // Rebuild the point ourselves along the exact off-axis ray (getWorldPointAsync would unproject
+    // with the component's symmetric projection, not this one). PlayCanvas's view matrix is built
+    // from the camera's world POSITION and ROTATION only — the rig node's scale is not in it — so
+    // view distances are world units and the ray must use the UNSCALED world rotation.
+    const view = { projectionMatrix: e.proj, transformMatrix: unscaled(cam.getWorldTransform().data) };
+    const Mw = view.transformMatrix;
+    const f = [-Mw[8], -Mw[9], -Mw[10]];
+    const zNear = cam.camera.nearClip, zFar = cam.camera.farClip;
+    const samples = cursorFootprint(uv[0], uv[1], this.height, this.pointer.aspect())
+      .filter(([u, v]) => u >= 0 && u <= 1 && v >= 0 && v <= 1)
+      .map(([u, v]) =>
+        picker.getPointDepthAsync(Math.min(W - 1, Math.floor(u * W)), Math.min(H - 1, Math.floor(v * H))).then((lin) => {
+          if (lin === null || !(lin >= 0 && lin < 1)) return null;
+          const z = zNear + lin * (zFar - zNear);
+          const ray = cursorViewRay(view, u, v);
+          return { x: ray.origin[0] + ray.direction[0] * z, y: ray.origin[1] + ray.direction[1] * z, z: ray.origin[2] + ray.direction[2] * z };
+        }),
+      );
+    this._inFlight = true;
+    const id = ++this._pickId;
+    this._pickStart = globalThis.performance ? globalThis.performance.now() : Date.now();
+    Promise.all(samples)
+      .then((pts) => {
+        if (this._disposed || id !== this._pickId) return;
+        let best = Infinity;
+        let nearest = null;
+        for (const p of pts) {
+          if (!p) continue;
+          const d = p.x * f[0] + p.y * f[1] + p.z * f[2];
+          if (d < best) {
+            best = d;
+            nearest = [p.x, p.y, p.z];
+          }
+        }
+        this.hit = nearest;
+      })
+      .catch(() => {
+        if (id === this._pickId) this.hit = null;
+      })
+      .finally(() => {
+        if (id === this._pickId) this._inFlight = false;
+      });
+  }
+}
+
+/** A copy of a column-major transform with unit rotation columns (scale removed). */
+function unscaled(m) {
+  const o = Array.from(m);
+  for (const c of [0, 4, 8]) {
+    const l = Math.hypot(o[c], o[c + 1], o[c + 2]) || 1;
+    o[c] /= l;
+    o[c + 1] /= l;
+    o[c + 2] /= l;
+  }
+  return o;
+}
+
+/** Column-major 4x4 multiply, a × b. */
+function mul4(a, b) {
+  const o = new Array(16);
+  for (let c = 0; c < 4; c++) {
+    for (let r = 0; r < 4; r++) {
+      let s = 0;
+      for (let k = 0; k < 4; k++) s += a[k * 4 + r] * b[c * 4 + k];
+      o[c * 4 + r] = s;
+    }
+  }
+  return o;
+}
+
+/** Unit quaternion [x, y, z, w] from a column-major rigid transform's rotation. */
+function quatFromMatrix(m) {
+  const m00 = m[0], m11 = m[5], m22 = m[10];
+  const tr = m00 + m11 + m22;
+  let x, y, z, w;
+  if (tr > 0) {
+    const s = Math.sqrt(tr + 1) * 2;
+    w = 0.25 * s;
+    x = (m[6] - m[9]) / s;
+    y = (m[8] - m[2]) / s;
+    z = (m[1] - m[4]) / s;
+  } else if (m00 > m11 && m00 > m22) {
+    const s = Math.sqrt(1 + m00 - m11 - m22) * 2;
+    w = (m[6] - m[9]) / s;
+    x = 0.25 * s;
+    y = (m[4] + m[1]) / s;
+    z = (m[8] + m[2]) / s;
+  } else if (m11 > m22) {
+    const s = Math.sqrt(1 + m11 - m00 - m22) * 2;
+    w = (m[8] - m[2]) / s;
+    x = (m[4] + m[1]) / s;
+    y = 0.25 * s;
+    z = (m[9] + m[6]) / s;
+  } else {
+    const s = Math.sqrt(1 + m22 - m00 - m11) * 2;
+    w = (m[1] - m[4]) / s;
+    x = (m[8] + m[2]) / s;
+    y = (m[9] + m[6]) / s;
+    z = 0.25 * s;
+  }
+  const l = Math.hypot(x, y, z, w) || 1;
+  return [x / l, y / l, z / l, w / l];
+}