@realitycollective/webxr-interactions 0.1.1-preview.2 → 0.1.1-preview.3

package/dist/runtime.js

@@ -13,14 +13,14 @@
  * unregister, enable and disable; behaviours disabled by capability
  * negotiation announce themselves with a `behaviourDisabled` event.
  */
-import { SELECT_PRESS_THRESHOLD, SELECT_RELEASE_THRESHOLD, unmetRequirements, } from "@realitycollective/webxr-input";
+import { SELECT_PRESS_THRESHOLD, SELECT_RELEASE_THRESHOLD, rayPoseFromRay, resolveEyeGazeOptions, unmetRequirements, } from "@realitycollective/webxr-input";
 import { createBehaviour, } from "./behaviours/factory.js";
 import { GrabBehaviour } from "./behaviours/grab.js";
 import { Emitter } from "./events.js";
-import { DwellState, resolveDwellConfig, DWELL_DEFAULTS } from "./gaze.js";
-import { vApplyQuat } from "./math.js";
+import { DwellState, GazeConsensus, resolveDwellConfig, DWELL_DEFAULTS } from "./gaze.js";
+import { quatConjugate, quatMultiply, vApplyQuat, vLength, vNormalize, vSub } from "./math.js";
+import { pickActivePointer, pointerVisualsFor, resolveNearPointerOptions, TouchPointerState, } from "./near-pointer.js";
 import { VelocityTracker } from "./velocity-tracker.js";
-const DEFAULT_POKE_RADIUS = 0.05;
 /** Interactor id used when gaze is synthesized from the head pose. */
 export const HEAD_GAZE_INTERACTOR_ID = "head-gaze";
 export class InteractionRuntime {
@@ -32,14 +32,27 @@ export class InteractionRuntime {
     events = new Emitter();
     feedbackEmitter = new Emitter();
     afterSample = new Emitter();
+    visualsEmitter = new Emitter();
+    nearPointer;
+    lastVisuals = [];
     velocityTracker;
     lastSources = new Map();
     capabilities;
     unsubscribeCaps;
     disposed = false;
+    eyeGaze;
+    gazeConsensus;
+    /** The eye-gaze source's target this frame: cone, consensus and suppression applied. */
+    eyeGazeTarget = null;
+    /** Where the eye-gaze ray last aimed at each target, for the pinching hand to aim at. */
+    gazePoints = new Map();
+    gazeHold = null;
     constructor(options) {
         this.provider = options.provider;
         this.hitTester = options.hitTester ?? null;
+        this.nearPointer = resolveNearPointerOptions(options.nearPointer);
+        this.eyeGaze = resolveEyeGazeOptions(options.eyeGaze);
+        this.gazeConsensus = new GazeConsensus(this.eyeGaze.dwellWindowSeconds);
         const d = options.dwellDefaults;
         this.dwellDefaults = {
             holdSeconds: d?.holdSeconds ?? DWELL_DEFAULTS.holdSeconds,
@@ -72,7 +85,7 @@ export class InteractionRuntime {
             })),
             transform: ports.transform,
             enabled: def.enabled !== false,
-            pokeRadius: def.pokeRadius ?? DEFAULT_POKE_RADIUS,
+            pokeRadius: def.pokeRadius ?? this.nearPointer.touchHoverEnter,
             gazeRequired: def.gaze?.required === true,
             dwellConfig: resolveDwellConfig(def.gaze?.dwell, this.dwellDefaults),
             dwell: new DwellState(),
@@ -177,12 +190,41 @@ export class InteractionRuntime {
     getProvider() {
         return this.provider;
     }
+    /** The near-pointer distances in force (the defaults, or the options given). */
+    getNearPointerOptions() {
+        return this.nearPointer;
+    }
+    /**
+     * The pointer that owns a source this frame: `"touch"`, `"grab"`, `"ray"`
+     * or `"gaze"`, or null when no pointer has a candidate (or the source is
+     * unknown). Decided by the core rule in `near-pointer.ts`, on every
+     * platform; a provider hint names the target but never the pointer.
+     */
+    getActivePointer(sourceId) {
+        return this.sourceStates.get(sourceId)?.active ?? null;
+    }
+    /**
+     * What to draw for every source this frame: the ray only while the ray is
+     * the active pointer or nothing is, the cursor at the active pointer's
+     * candidate. Published once per `update`, after the sources; a binding
+     * applies it to its ray and cursor visuals, and a native binding hands it
+     * to the host as `applyPointerVisuals`. The list and its entries are fresh
+     * each frame and never written to again.
+     */
+    onPointerVisuals(listener) {
+        return this.visualsEmitter.subscribe(listener);
+    }
+    /** The visuals last published for a source, or undefined before its first frame. */
+    getPointerVisuals(sourceId) {
+        return this.lastVisuals.find((visuals) => visuals.sourceId === sourceId);
+    }
     // -- frame ------------------------------------------------------------------
     update(dt) {
         if (this.disposed)
             return;
         const sampled = this.provider.sample();
-        const sources = this.velocityTracker ? this.velocityTracker.update(sampled, dt) : sampled;
+        const tracked = this.velocityTracker ? this.velocityTracker.update(sampled, dt) : sampled;
+        const sources = this.followHand(tracked);
         const hints = this.provider.sampleHints?.() ?? [];
         this.lastSources = new Map(sources.map((source) => [source.id, source]));
         this.afterSample.emit(sources);
@@ -194,10 +236,31 @@ export class InteractionRuntime {
             else
                 hintBySource.set(hint.sourceId, [hint]);
         }
+        // With eye gaze live, the hands and controllers go first: whether one of
+        // them has a near pointer active this frame decides what gaze may target
+        // (IWSDK reads the hand pointers' state before it drives the gaze pointer).
+        const eyeGazeLive = this.capabilities.eyeGaze;
+        const ordered = eyeGazeLive
+            ? [...sources].sort((a, b) => Number(a.kind === "gaze") - Number(b.kind === "gaze"))
+            : sources;
+        this.eyeGazeTarget = null;
+        let sawGaze = false;
         const seen = new Set();
-        for (const source of sources) {
+        const visuals = [];
+        for (const source of ordered) {
             seen.add(source.id);
+            if (eyeGazeLive && source.kind === "gaze") {
+                sawGaze = true;
+                this.resolveEyeGaze(source, dt);
+            }
             this.updateSource(source, hintBySource.get(source.id) ?? []);
+            const state = this.sourceState(source.id);
+            visuals.push(pointerVisualsFor(source.id, source.ray !== undefined, state.active, state.candidate));
+        }
+        if (!sawGaze) {
+            // No eye-gaze source this frame (none, or far targeting handed back).
+            this.gazeHold = null;
+            this.gazeConsensus.reset();
         }
         // Sources that vanished mid-hold release everything they held.
         for (const [id, state] of this.sourceStates) {
@@ -208,6 +271,8 @@ export class InteractionRuntime {
         }
         this.updateGaze(sources, dt);
         this.tickBehaviours(sources, dt);
+        this.lastVisuals = visuals;
+        this.visualsEmitter.emit(visuals);
     }
     dispose() {
         if (this.disposed)
@@ -231,12 +296,129 @@ export class InteractionRuntime {
         this.sourceStates.clear();
         this.lastSources.clear();
         this.velocityTracker?.reset();
+        this.gazeHold = null;
+        this.gazePoints.clear();
+        this.gazeConsensus.reset();
+        this.lastVisuals = [];
+    }
+    // -- eye gaze -----------------------------------------------------------------
+    /**
+     * The targeting half of the eye-gaze rule (`@realitycollective/webxr-input`
+     * `eye-gaze.ts` states the whole of it): the cone's best candidate this
+     * frame, then the dwell consensus over the window, and nothing at all while
+     * a hand's near pointer is hovering or selecting (IWSDK
+     * `suppressWhenDirectPointerActive`). A frame with no valid gaze ray clears
+     * the record, as IWSDK resets its conecaster, unless a selection is held.
+     */
+    resolveEyeGaze(source, dt) {
+        const ray = source.ray;
+        if (!ray) {
+            if (!this.gazeHold)
+                this.gazeConsensus.reset();
+            this.eyeGazeTarget = null;
+            return;
+        }
+        if (this.eyeGaze.suppressWhenDirectPointerActive && this.directPointerActive()) {
+            this.gazeConsensus.skip(dt);
+            this.eyeGazeTarget = null;
+            return;
+        }
+        const halfAngle = (this.eyeGaze.coneAngleDegrees * Math.PI) / 180;
+        const maxLength = this.eyeGaze.maxRayLength;
+        let best = null;
+        if (this.hitTester) {
+            best = this.hitTester.hitCone
+                ? this.hitTester.hitCone(ray, halfAngle, maxLength)
+                : this.hitTester.hitRay(ray);
+            if (best && (best.distance > maxLength || !this.targetable(best.interactableId)))
+                best = null;
+        }
+        if (best)
+            this.gazePoints.set(best.interactableId, best.point);
+        this.eyeGazeTarget = this.gazeConsensus.update(best?.interactableId ?? null, dt, (id) => this.targetable(id));
+    }
+    /** A hand's near pointer is hovering or selecting: touch or grab owns the hand, or a grab is held. */
+    directPointerActive() {
+        for (const [id, state] of this.sourceStates) {
+            const source = this.lastSources.get(id);
+            if (!source || (source.handedness !== "left" && source.handedness !== "right"))
+                continue;
+            if (state.grabbed)
+                return true;
+            if ((state.active === "touch" || state.active === "grab") && (state.hoverTarget !== null || state.pressTarget !== null))
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Once a gaze selection is held, the pinching hand drives the pointer
+     * (IWSDK `pointerTransformFollowsHand`): the frame the hold starts, a ray
+     * from the hand's ray space is aimed at the gaze hit and its orientation
+     * relative to the hand recorded; every later frame the gaze snapshot's ray
+     * is that hand's ray space turned by the same amount. When the hand could
+     * not reach the hit, or the option is off, the ray stays where the
+     * selection was made. Consumers of `onSample` and `getSource`, and every
+     * behaviour, see the followed ray.
+     */
+    followHand(sources) {
+        const hold = this.gazeHold;
+        if (!hold || !this.capabilities.eyeGaze)
+            return sources;
+        const index = sources.findIndex((source) => source.id === hold.sourceId);
+        if (index < 0)
+            return sources;
+        const source = sources[index];
+        let ray = hold.ray;
+        if (hold.relative && source.selectorPose) {
+            const quaternion = quatMultiply(source.selectorPose.quaternion, hold.relative);
+            ray = { origin: [...source.selectorPose.position], direction: vApplyQuat([0, 0, -1], quaternion) };
+        }
+        const out = [...sources];
+        out[index] = { ...source, ray };
+        return out;
+    }
+    /** Start, keep or end the gaze hold from the gaze source's press and grab state this frame. */
+    trackGazeHold(source, state, target) {
+        if (!state.pressed && !state.grabbed) {
+            this.gazeHold = null;
+            return;
+        }
+        if (this.gazeHold?.sourceId === source.id)
+            return;
+        if (!source.ray)
+            return;
+        let relative = null;
+        const point = target ? this.gazePoints.get(target) : undefined;
+        const selector = source.selectorPose;
+        if (this.eyeGaze.pointerTransformFollowsHand && selector && point) {
+            const aim = vSub(point, selector.position);
+            if (vLength(aim) > 1e-6) {
+                const up = vApplyQuat([0, 1, 0], selector.quaternion);
+                const aimed = rayPoseFromRay({ origin: selector.position, direction: vNormalize(aim) }, up);
+                const aimedRay = { origin: aimed.position, direction: vApplyQuat([0, 0, -1], aimed.quaternion) };
+                // IWSDK follows the hand only when the ray from the hand reaches the target.
+                const reaches = this.hitTester ? this.hitTester.hitRay(aimedRay) !== null : true;
+                if (reaches)
+                    relative = quatMultiply(quatConjugate(selector.quaternion), aimed.quaternion);
+            }
+        }
+        this.gazeHold = { sourceId: source.id, ray: source.ray, relative };
     }
     // -- internals ---------------------------------------------------------------
     sourceState(id) {
         let state = this.sourceStates.get(id);
         if (!state) {
-            state = { hoverTarget: null, pressed: false, pressTarget: null, grabbed: false, grabTarget: null };
+            state = {
+                hoverTarget: null,
+                resolvedBy: null,
+                pressed: false,
+                pressTarget: null,
+                grabbed: false,
+                grabTarget: null,
+                active: null,
+                candidate: null,
+                touch: new TouchPointerState(),
+            };
             this.sourceStates.set(id, state);
         }
         return state;
@@ -260,46 +442,105 @@ export class InteractionRuntime {
             info.angularVelocity = source.angularVelocity;
         return info;
     }
-    /** Resolve this source's target: hints beat poke beats ray. */
-    resolveTarget(source, hints) {
+    /**
+     * Resolve this source's target and its active pointer. The pointer is the
+     * core rule (`near-pointer.ts`): touch, then grab, then ray, the first with
+     * a candidate, locked while it presses or grabs. The target is a provider
+     * hint when there is one (provider power: IWSDK's mesh-accurate tags), else
+     * the active pointer's candidate. An eye-gaze source targets through the
+     * cone and consensus instead.
+     */
+    resolveTarget(source, hints, state) {
+        state.resolvedBy = null;
         const hint = hints.find((h) => h.state === "grab") ??
             hints.find((h) => h.state === "press") ??
             hints.find((h) => h.state === "hover");
-        if (hint && this.targetable(hint.targetId))
-            return hint.targetId;
-        if (source.indexTip && this.hitTester) {
-            // One proximity query at the largest radius any interactable asked for,
-            // then the nearest hit must be inside ITS OWN radius. A per-interactable
-            // query would be a hit test per registration per source per frame; this
-            // keeps it to one. The trade: a farther interactable with a bigger radius
-            // is not found behind a nearer one with a smaller radius, because the
-            // tester returns only the nearest.
-            const poke = this.hitTester.hitProximity(source.indexTip, this.maxPokeRadius());
-            if (poke && this.targetable(poke.interactableId)) {
-                const registered = this.interactables.get(poke.interactableId);
-                if (poke.distance <= registered.pokeRadius)
-                    return poke.interactableId;
+        if (this.capabilities.eyeGaze && source.kind === "gaze") {
+            state.active = "gaze";
+            state.touch.reset();
+            const gazeTarget = this.eyeGazeTarget;
+            const point = gazeTarget ? this.gazePoints.get(gazeTarget) : undefined;
+            state.candidate = gazeTarget && point ? { targetId: gazeTarget, point, distance: 0 } : null;
+            if (hint && this.targetable(hint.targetId)) {
+                state.resolvedBy = "hint";
+                return hint.targetId;
             }
+            if (gazeTarget !== null)
+                state.resolvedBy = "gaze";
+            return gazeTarget;
         }
-        if (source.ray && this.hitTester) {
-            const hit = this.hitTester.hitRay(source.ray);
-            if (hit && this.targetable(hit.interactableId))
-                return hit.interactableId;
+        const touch = this.touchCandidate(source, state);
+        const grab = this.grabCandidate(source);
+        const ray = this.rayCandidate(source);
+        const selecting = state.pressTarget !== null || state.grabTarget !== null;
+        const active = pickActivePointer({ touch: touch !== null, grab: grab !== null, ray: ray !== null }, state.active, selecting);
+        state.active = active;
+        state.candidate = active === "touch" ? touch : active === "grab" ? grab : active === "ray" ? ray : null;
+        if (hint && this.targetable(hint.targetId)) {
+            state.resolvedBy = "hint";
+            return hint.targetId;
         }
-        return null;
+        if (!state.candidate || !active)
+            return null;
+        state.resolvedBy = active === "touch" ? "poke" : active;
+        return state.candidate.targetId;
+    }
+    /**
+     * The touch pointer's candidate this frame, advancing the source's touch
+     * machine: the nearest target within the widest touch band, hovering by
+     * that target's own enter and exit distances. One proximity query at the
+     * largest radius any interactable asked for, then the nearest hit must be
+     * inside ITS OWN band; the trade is that a farther target with a wider
+     * band is not found behind a nearer one with a narrower band.
+     */
+    touchCandidate(source, state) {
+        const { touchHoverEnter, touchHoverExit, touchDown } = this.nearPointer;
+        const hysteresis = touchHoverExit - touchHoverEnter;
+        if (!source.indexTip || !this.hitTester) {
+            state.touch.update(null, touchHoverEnter, touchHoverExit, touchDown);
+            return null;
+        }
+        const hit = this.hitTester.hitProximity(source.indexTip, this.touchQueryRadius());
+        const registered = hit ? this.interactables.get(hit.interactableId) : undefined;
+        if (!hit || !registered || !registered.enabled) {
+            state.touch.update(null, touchHoverEnter, touchHoverExit, touchDown);
+            return null;
+        }
+        const enter = registered.pokeRadius;
+        const update = state.touch.update(hit.distance, enter, enter + hysteresis, touchDown);
+        return update.hovering ? { targetId: hit.interactableId, point: hit.point, distance: hit.distance } : null;
+    }
+    /** The grab pointer's candidate: the nearest target whose surface the grip sphere reaches. */
+    grabCandidate(source) {
+        if (!source.gripPose || !this.hitTester)
+            return null;
+        const hit = this.hitTester.hitProximity(source.gripPose.position, this.nearPointer.grabRadius);
+        if (!hit || !this.targetable(hit.interactableId))
+            return null;
+        return { targetId: hit.interactableId, point: hit.point, distance: hit.distance };
+    }
+    /** The ray pointer's candidate: the nearest target along the ray. */
+    rayCandidate(source) {
+        if (!source.ray || !this.hitTester)
+            return null;
+        const hit = this.hitTester.hitRay(source.ray);
+        if (!hit || !this.targetable(hit.interactableId))
+            return null;
+        return { targetId: hit.interactableId, point: hit.point, distance: hit.distance };
     }
     targetable(id) {
         const registered = this.interactables.get(id);
         return registered !== undefined && registered.enabled;
     }
-    /** The largest poke radius among enabled interactables; the default when none is registered. */
-    maxPokeRadius() {
+    /** The widest touch exit band among enabled interactables; the default exit distance when none is registered. */
+    touchQueryRadius() {
+        const hysteresis = this.nearPointer.touchHoverExit - this.nearPointer.touchHoverEnter;
         let radius = 0;
         for (const registered of this.interactables.values()) {
             if (registered.enabled && registered.pokeRadius > radius)
                 radius = registered.pokeRadius;
         }
-        return radius > 0 ? radius : DEFAULT_POKE_RADIUS;
+        return radius > 0 ? radius + hysteresis : this.nearPointer.touchHoverExit;
     }
     flags(registered) {
         let pressable = false;
@@ -316,7 +557,7 @@ export class InteractionRuntime {
     }
     updateSource(source, hints) {
         const state = this.sourceState(source.id);
-        const target = this.resolveTarget(source, hints);
+        const target = this.resolveTarget(source, hints, state);
         // Hover transitions.
         if (state.hoverTarget !== target) {
             if (state.hoverTarget)
@@ -335,8 +576,11 @@ export class InteractionRuntime {
         const grabSignal = Math.max(source.squeeze, selectAsGrab ? source.select : 0);
         const hintPress = hints.some((h) => h.state === "press");
         const hintGrab = hints.some((h) => h.state === "grab") || source.nativeGrabbing === true;
-        // Press with hysteresis (hints override).
-        const pressActive = hintPress || (state.pressed
+        // The touch press: the fingertip at or under the press distance while
+        // touch owns the source (`near-pointer.ts`), with no select needed.
+        const touchPress = state.active === "touch" && state.touch.isPressed;
+        // Press with hysteresis (hints and the touch press override).
+        const pressActive = hintPress || touchPress || (state.pressed
             ? pressSignal > SELECT_RELEASE_THRESHOLD
             : pressSignal >= SELECT_PRESS_THRESHOLD);
         if (pressActive && !state.pressed) {
@@ -384,6 +628,8 @@ export class InteractionRuntime {
                 state.grabTarget = null;
             }
         }
+        if (this.capabilities.eyeGaze && source.kind === "gaze")
+            this.trackGazeHold(source, state, target);
     }
     gazeAllows(registered) {
         return !registered.gazeRequired || registered.gazeHovered;
@@ -452,6 +698,9 @@ export class InteractionRuntime {
             this.endGrab(state.grabTarget, sourceId, info);
         if (state.hoverTarget)
             this.setHover(state.hoverTarget, sourceId, false);
+        state.touch.reset();
+        state.active = null;
+        state.candidate = null;
     }
     releaseAllHolds(id) {
         for (const [sourceId, state] of this.sourceStates) {
@@ -475,7 +724,16 @@ export class InteractionRuntime {
         let gazeRay = null;
         let gazeId = HEAD_GAZE_INTERACTOR_ID;
         const gazeSource = sources.find((s) => s.kind === "gaze");
-        if (gazeSource?.ray) {
+        // Eye gaze, when live, is the gaze for gating and dwell too, with the
+        // target the cone and consensus chose (nothing while a near pointer is
+        // active, or while the pose is invalid). Otherwise a head-gaze source, or
+        // the head pose, targets by plain ray as before.
+        const eyeGaze = this.capabilities.eyeGaze && gazeSource !== undefined;
+        if (eyeGaze) {
+            gazeRay = gazeSource.ray ?? null;
+            gazeId = gazeSource.id;
+        }
+        else if (gazeSource?.ray) {
             gazeRay = gazeSource.ray;
             gazeId = gazeSource.id;
         }
@@ -486,7 +744,11 @@ export class InteractionRuntime {
                 direction: vApplyQuat([0, 0, -1], head.quaternion),
             };
         }
-        const gazeTarget = gazeRay && this.hitTester ? this.hitTester.hitRay(gazeRay)?.interactableId ?? null : null;
+        const gazeTarget = eyeGaze
+            ? this.eyeGazeTarget
+            : gazeRay && this.hitTester
+                ? (this.hitTester.hitRay(gazeRay)?.interactableId ?? null)
+                : null;
         for (const [id, registered] of this.interactables) {
             const hovered = registered.enabled && gazeTarget === id;
             if (hovered !== registered.gazeHovered) {