@displayxr/inline3d 1.49.2 → 1.50.0

package/hands.d.ts

@@ -0,0 +1,198 @@
+// Type definitions for @displayxr/inline3d/hands.
+// EXPERIMENTAL — not covered by the 1.x semver promise. See docs/sdk-stability.md.
+//
+// Tracked hands on an inline-3D session (WebXR Hand Input, DisplayXR Browser 1.8.0+ on Windows
+// with a hand provider), and the pinch gestures that drive a viewer's orbit from them. Inert
+// everywhere else: `watchHands` reports no hands and `handControls` moves nothing.
+
+import type { Inline3D, Inline3DUnsupported } from './index.js';
+
+/** The 25 WebXR hand joint names, in `XRHand` iteration order. */
+export const HAND_JOINTS: ReadonlyArray<string>;
+
+/** Gesture tuning. Metres in the joints' frame (the rig frame), radians. */
+export interface HandGestureConfig {
+  /** Thumb-tip to index-tip distance that starts a pinch, for a hand of `referencePalmM` (default 0.025). */
+  pinchEnterM: number;
+  /** Distance that ends it, > enter (default 0.04). */
+  pinchExitM: number;
+  /**
+   * Palm length every distance here is stated for; a sample's own palm length scales them (pinch
+   * thresholds, orbit gain, separation guard), so a scaled hand behaves the same. 0 = never scale
+   * (default 0.095).
+   */
+  referencePalmM: number;
+  /** Radians of orbit per metre of pinch-point drag, for a reference-size hand (default 10). */
+  orbitRadPerM: number;
+  /** EMA time constant on the pinch point, seconds; 0 = off (default 0.05). */
+  smoothTauS: number;
+  /** Per-frame zoom ratio clamp, both ways (default 1.25). */
+  maxZoomStep: number;
+  /** Two-hand gestures are ignored while the hands are further apart than this (default 1.2). */
+  maxHandSeparationM: number;
+}
+export const HAND_GESTURE_DEFAULTS: Readonly<HandGestureConfig>;
+
+/** Two-threshold pinch state. */
+export declare class PinchDetector {
+  readonly pinched: boolean;
+  reset(): void;
+  /** `tracked: false` releases regardless of distance. Returns the new state. */
+  update(tracked: boolean, tipDistance: number, enter: number, exit: number): boolean;
+}
+
+/** One hand's input to the gesture controller. Positions `[x, y, z]`, metres. */
+export interface HandGestureSample {
+  tracked?: boolean;
+  thumbTip: ArrayLike<number>;
+  indexTip: ArrayLike<number>;
+  /** Wrist → middle knuckle, metres; 0 / absent = unknown (the metre thresholds apply as given). */
+  palmLength?: number;
+}
+
+/**
+ * The pose delta for one frame, in the SUBJECT's convention (what `SceneViewer.nudgePose` takes):
+ * yaw / pitch / roll in radians to ADD, zoom a ratio to MULTIPLY. A pinch dragged right → yaw +;
+ * dragged up → pitch −; two hands turning counter-clockwise → roll +; spreading → zoom > 1.
+ */
+export interface HandGestureDelta {
+  yaw: number;
+  pitch: number;
+  roll: number;
+  zoom: number;
+  /** Exactly one hand pinched this frame. */
+  oneHand: boolean;
+  /** Both hands pinched this frame. */
+  twoHand: boolean;
+  any: boolean;
+}
+
+/**
+ * Pinch gestures → pose deltas, per frame. Pure arithmetic (no DOM, no WebXR): a port of the
+ * DisplayXR Model Viewer's controls. Hand-relative pinch (thresholds scale by palm length), one
+ * pinched hand orbits, two pinched hands zoom (separation ratio), roll (twist in the display
+ * plane) and orbit (midpoint drag). The first frame of a gesture anchors; a hand gained or lost
+ * re-anchors, so a mode change never jumps.
+ */
+export declare class HandGestureController {
+  constructor(config?: Partial<HandGestureConfig>);
+  config: HandGestureConfig;
+  update(frame: {
+    /** Seconds, any origin (drives the EMA). */
+    time?: number;
+    left?: HandGestureSample | null;
+    right?: HandGestureSample | null;
+    /** Multiplies every position first (default 1). */
+    lengthScale?: number;
+  }): HandGestureDelta;
+  /** Pinch state after the last update: `0` / `'left'`, `1` / `'right'`. */
+  pinched(hand: 0 | 1 | 'left' | 'right'): boolean;
+  /** Last thumb-index tip distance, metres; < 0 when untracked. */
+  tipDistance(hand: 0 | 1 | 'left' | 'right'): number;
+  /** Forget every anchor. */
+  reset(): void;
+}
+
+/** Can this document deliver hand sources at all (`XRHand` present)? */
+export function handInputAvailable(): boolean;
+
+/**
+ * The browser's per-frame hand gate: `'available'`, a withheld reason (`'camera-rig'`,
+ * `'posed-rig'`, `'rig-factors'`, `'no-canvas'`, `'not-fullscreen'`, `'scaled-rig'`), or
+ * `'unknown'` where the session has no such surface.
+ */
+export function handGate(session: XRSession | null | undefined): string;
+
+/** One tracked hand as `watchHands` reports it. Positions `[x, y, z]` metres in the watch's reference space. */
+export interface HandSample {
+  readonly tracked: boolean;
+  readonly handedness: 'left' | 'right' | 'none';
+  /** 25 × 16 column-major joint transforms (`XRFrame.fillPoses`). */
+  readonly poses: Float32Array;
+  /** 25 joint radii, metres. */
+  readonly radii: Float32Array;
+  /** Position of a joint by WebXR name, or null for an unknown name. */
+  joint(name: string): number[] | null;
+  readonly wrist: number[];
+  readonly thumbTip: number[];
+  readonly indexTip: number[];
+  /** Wrist → middle knuckle, metres. */
+  readonly palmLength: number;
+  readonly source: XRInputSource;
+}
+
+export interface HandsFrame {
+  /** Seconds (the session frame time / 1000). */
+  time: number;
+  /** As `handGate()`. */
+  gate: string;
+  left: HandSample | null;
+  right: HandSample | null;
+  /** Hands tracked this frame. */
+  hands: number;
+  frame: XRFrame;
+}
+
+export interface HandsWatch {
+  stop(): void;
+  readonly running: boolean;
+  readonly state: { gate: string; hands: number; supported: boolean };
+}
+
+export interface WatchHandsOptions {
+  /** Joints are located in this space (default: the wall's `refSpace`). */
+  referenceSpace?: XRReferenceSpace;
+  /** `inputsourceschange`, hand sources only. */
+  onChange?: (e: { added: XRInputSource[]; removed: XRInputSource[] }) => void;
+}
+
+/**
+ * Watch the hands on an inline-3D session: `onFrame` every session frame with the tracked hands
+ * (or none). Runs its own `session.requestAnimationFrame` loop beside the SDK's; stops itself
+ * when the session ends.
+ */
+export function watchHands(
+  wall: Inline3D | Inline3DUnsupported | null | undefined,
+  onFrame: (f: HandsFrame) => void,
+  opts?: WatchHandsOptions,
+): HandsWatch;
+
+export interface HandControlOptions extends WatchHandsOptions {
+  /** Controller tuning. */
+  gestures?: Partial<HandGestureConfig>;
+  /** Multiplies every hand position before the gesture math (default 1). */
+  lengthScale?: number;
+  /** Apply the two-hand roll where the viewer has a roll axis (default true). */
+  roll?: boolean;
+  /** Apply the two-hand zoom (default true). */
+  zoom?: boolean;
+  /** After each frame: the hands frame and the gesture delta. */
+  onFrame?: (f: HandsFrame, d: HandGestureDelta) => void;
+}
+
+/** What `handControls` returns, and what `handle.hands` is on an `addModel` / `addSplat` handle. */
+export interface HandControls {
+  stop(): void;
+  readonly running: boolean;
+  readonly controller: HandGestureController;
+  readonly state: { gate: string; hands: number; supported: boolean };
+}
+
+/** Anything with the viewer pose surface: a SceneViewer, the PlayCanvas viewer, or a handle whose `viewer` is one. */
+export interface HandControlTarget {
+  nudgePose?(delta: { yaw?: number; pitch?: number; zoom?: number; roll?: number }): void;
+  getPose?(opts?: { target?: boolean }): { yaw: number; pitch: number; zoom: number; roll?: number };
+  setPose?(pose: { yaw?: number; pitch?: number; zoom?: number; roll?: number }): void;
+  readonly viewer?: HandControlTarget | null;
+}
+
+/**
+ * Drive a viewer's orbit from the hands: one pinch dragged turns it, two pinches spread / twist
+ * / dragged zoom / roll / turn it. Prefers `viewer.nudgePose()` (eased, counts as input for the
+ * idle turntable), falls back to `getPose` + `setPose`. Pointer and wheel keep working.
+ */
+export function handControls(
+  wall: Inline3D | Inline3DUnsupported | null | undefined,
+  target: HandControlTarget,
+  opts?: HandControlOptions,
+): HandControls;

package/js/camera/capture.js

@@ -41,6 +41,7 @@
 // `(stream, info) => MediaStream | Promise<MediaStream>`, returning a rectified SBS stream.
 
 import { looksSbs } from './geometry.js';
+import { isHalfSbsCamera, unsqueezeHalfSbs } from './half-sbs.js';
 import { isAppShareDevice, isAppShareHint } from './app-share.js';
 
 const MONO_CONSTRAINTS = { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } };
@@ -213,6 +214,14 @@ export async function openCamera(want = 'auto', o = {}) {
     // or — on a browser without the app-share device patch, which lists the app as a "3D Camera
     // (DisplayXR) - <app>" — baseline 0 and FOV 0, which no real camera advertises. Every probe
     // path checks this on the OPENED track (rejectOpened), since the label cannot tell them apart.
+    // A half-SBS webcam: send true side-by-side eyes, whichever path opened it. 'mono' stays raw.
+    if (want !== 'mono' && isHalfSbsCamera(t.label)) {
+      const pair = await unsqueezeHalfSbs(t);
+      if (pair) {
+        log('camera-half-sbs', { label: t.label, width: pair.width, height: pair.height });
+        return { ...pair, deviceId: st.deviceId || null, label: t.label, stereo: null, appShare: false };
+      }
+    }
     const appShare = isAppShareDevice({ label: t.label || '', deviceId: st.deviceId || '' }) || isAppShareHint(st.displayxrStereo);
     return { stream: s, track: t, width: st.width, height: st.height, deviceId: st.deviceId || null, label: t.label || '', stereo, appShare };
   };

package/js/camera/half-sbs.js

@@ -0,0 +1,83 @@
+// camera/half-sbs.js — USB stereo webcams that send a half-width side-by-side frame: one 16:9
+// image whose left and right halves are each an eye squeezed 2:1 in width.
+//
+// Shape cannot identify these: a 3840x2160 half-SBS frame looks exactly like a 4K webcam's, so they
+// are listed by USB id. Asking one for a wide mode does not help either: the browser reaches
+// 3840x1080 by cropping the middle rows of 3840x2160, which still reads as a pair but leaves each
+// eye 2x too tall with half the vertical view. So the camera is read in its native mode and every
+// frame is redrawn, eyes un-squeezed, into a true side-by-side track.
+
+/** USB `vvvv:pppp` ids of cameras that send half-SBS (the browser's label ends with the id). */
+export const HALF_SBS_USB_IDS = Object.freeze(['0733:011a']);
+
+/** The widest eye sent: the runtime's own output for such a camera (2560x720 side by side). */
+const MAX_EYE_WIDTH = 1280;
+const NATIVE_CONSTRAINTS = { width: { ideal: 3840 }, height: { ideal: 2160 }, frameRate: { ideal: 30 }, resizeMode: 'none' };
+
+/** Does this device label name a half-SBS camera? */
+export function isHalfSbsCamera(label) {
+  const l = String(label || '').toLowerCase();
+  return HALF_SBS_USB_IDS.some((id) => l.includes(`(${id})`));
+}
+
+/** The full-SBS frame size for a half-SBS source: square-pixel eyes, no upscaling, even numbers. */
+export function unsqueezedSize(width, height) {
+  const eyeW = Math.min(MAX_EYE_WIDTH, width);
+  const eyeH = Math.max(2, Math.round((eyeW * height) / width / 2) * 2);
+  return { width: eyeW * 2, height: eyeH };
+}
+
+/**
+ * Put `source` in its native mode and return a side-by-side track with the eyes un-squeezed, or
+ * null when that cannot be done here (the engine has no MediaStreamTrackProcessor or the camera
+ * refuses a 16:9 mode); the source is then left open as it is. Stopping the returned track stops
+ * the source.
+ *
+ * @param {MediaStreamTrack} source
+ * @returns {Promise<{stream: MediaStream, track: MediaStreamTrack, width: number, height: number}|null>}
+ */
+export async function unsqueezeHalfSbs(source) {
+  if (typeof MediaStreamTrackProcessor !== 'function' || typeof document === 'undefined') return null;
+  try {
+    await source.applyConstraints(NATIVE_CONSTRAINTS);
+  } catch {
+    return null;
+  }
+  const st = source.getSettings();
+  if (!(st.width > 0) || !(st.height > 0) || Math.abs(st.width / st.height - 16 / 9) > 0.01) return null;
+
+  const { width, height } = unsqueezedSize(st.width, st.height);
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext('2d', { alpha: false });
+  if (!ctx) return null;
+  const stream = canvas.captureStream(st.frameRate > 0 ? st.frameRate : 30);
+  const track = stream.getVideoTracks()[0];
+  const reader = new MediaStreamTrackProcessor({ track: source }).readable.getReader();
+
+  let stopped = false;
+  const stopTrack = track.stop.bind(track);
+  track.stop = () => {
+    stopped = true;
+    reader.cancel().catch(() => {});
+    source.stop();
+    stopTrack();
+  };
+  (async () => {
+    for (;;) {
+      const { value: frame, done } = await reader.read();
+      if (done) break;
+      ctx.drawImage(frame, 0, 0, width, height);
+      frame.close();
+    }
+  })()
+    .catch(() => {})
+    .finally(() => {
+      if (stopped) return;
+      // The camera went away: this track ends with it, like the device's own would.
+      track.stop();
+      track.dispatchEvent(new Event('ended'));
+    });
+  return { stream, track, width, height };
+}

package/js/inline3d-call.js

@@ -2565,7 +2565,7 @@ class SelfTile {
 
   /** For `call.diagnostics().self`: what the self view actually shows. */
   diag() {
-    if (!this.view) return { route: null, woven: false, reason: 'no-camera', firstWoven: null, layerRetries: 0 };
+    if (!this.view) return { route: null, woven: false, reason: 'no-camera', firstWoven: null, layerRetries: 0, frameAgeMs: null };
     return this.view.weaveState();
   }
 

package/js/inline3d-camera.js

@@ -564,6 +564,8 @@ class CameraView {
     this._scratch = null;
     this._luma = null; // the latest grayscale source copy (auto-convergence and the depth cursor read it)
     this.disparityPx = null;
+    this._lastFrameAt = 0; // performance.now() of the last NEW camera frame (0 = none yet)
+    this._lastVideoTime = -1;
     // `cursor: 'depth'` (ADR-046 on a pair): validated now, its module loaded only on opt-in.
     this._cursorOpt = resolvePairCursorOption(o.cursor, '@displayxr/inline3d/camera');
     /** The depth cursor once its module has loaded; null without `cursor: 'depth'`. */
@@ -598,9 +600,28 @@ class CameraView {
       reason: this.fallbackReason,
       firstWoven: this.route === 'woven-sbs' ? (this._fw ? { ...this._fw } : 'pending') : null,
       layerRetries: this._layerFails,
+      frameAgeMs: this.frameAgeMs,
     };
   }
 
+  /**
+   * Milliseconds since the camera last delivered a NEW frame to this view (its video's
+   * `currentTime` moved), or null before the first one. A view whose track stalls (a camera taken
+   * by another app, a suspended capture) keeps painting its last frame; this is how a page tells.
+   */
+  get frameAgeMs() {
+    return this._lastFrameAt ? Math.round(performance.now() - this._lastFrameAt) : null;
+  }
+
+  /** Note a new camera frame (the video's clock moved since the last paint). */
+  _noteFrame(v) {
+    if ((v.readyState || 0) < 2) return;
+    const t = v.currentTime;
+    if (t === this._lastVideoTime) return;
+    this._lastVideoTime = t;
+    this._lastFrameAt = performance.now();
+  }
+
   /** The per-eye convergence shift currently painted, source px. */
   get convergencePx() {
     return this.conv.current;
@@ -658,7 +679,14 @@ class CameraView {
   }
 
   _registerWoven() {
-    if (!this.buffer) this.buffer = document.createElement('canvas');
+    if (!this.buffer) {
+      // Empty until the camera's first frame is painted into it: the wall does not count an empty
+      // buffer as a stereo frame, so firstWoven cannot settle "woven" over a tile that never
+      // showed the camera (a track that delivers no frames).
+      this.buffer = document.createElement('canvas');
+      this.buffer.width = 0;
+      this.buffer.height = 0;
+    }
     this._paintWoven(); // the first frame exists before the layer does
     const handle = this.rectCover === undefined ? this.wall.addImage(this.canvas, this.buffer) : this.wall.addImage(this.canvas, this.buffer, { rectCover: this.rectCover });
     this.handle = handle;
@@ -737,6 +765,7 @@ class CameraView {
     if (this.removed) return;
     const v = this.cam.video;
     if (!v || this.cam.state === 'closed') return;
+    this._noteFrame(v);
     if (this.route === 'woven-sbs') this._paintWoven();
     else this._paintFlat(v);
   }