@tangram.gl/tangram-layers 0.0.0-stage → 0.0.0

package/src/experimental/webxr/presentation.ts

@@ -0,0 +1,457 @@
+// tangram-layers
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Matrix4} from '@math.gl/core';
+import {DoubleClickDrag, EventManager, Pan, Pinch, Tap} from 'mjolnir.js';
+import {createXRPlacementMatrix, getXRGlobeVisibleBounds} from './projection.js';
+import type {MapController, InteractionState, ViewStateChangeParameters} from '@deck.gl/core';
+import type {HostFrameOptions} from '@tangram.gl/tangram-renderer/core';
+import type {XRDeckController, XRDeckView, XRFrameState, XRInteractionIntent, XRPlacement,
+  XRHostFrameFields, XRPresentationFrame, XRPresentationMode, XRPresentationRenderView, XRViewState,
+  XRViewportSize, XRVector3} from './types.js';
+
+/** Initialization of the shared navigation state and independent room placement. */
+export type WebXRPresentationOptions = {
+  view: XRDeckView;
+  viewState: XRViewState;
+  placement?: XRPlacement;
+  mode?: XRPresentationMode;
+};
+
+/** Host input and callbacks passed to deck.gl's controller. */
+export type WebXRControllerOptions = {
+  element: HTMLElement;
+  timeline: ConstructorParameters<typeof MapController>[0]['timeline'];
+  onViewStateChange?: (parameters: ViewStateChangeParameters<XRViewState>) => void;
+  onStateChange?: (state: InteractionState) => void;
+};
+
+/** Input needed to prepare the next logical, stereo, or immersive frame. */
+export type WebXRFrameOptions = XRViewportSize & {
+  frameState?: XRFrameState;
+  mode?: XRPresentationMode;
+  interpupillaryDistance?: number;
+  /** Opt-in shared worker/cache budgets; never multiplied by the number of eyes. */
+  tileResources?: HostFrameOptions['tileResources'];
+};
+
+/** Default human interpupillary distance used by desktop stereo preview, in meters. */
+export const DEFAULT_INTERPUPILLARY_DISTANCE = 0.064;
+
+/**
+ * Expands one logical deck.gl view into mono, stereo-preview, or immersive render views.
+ *
+ * Placement is independent from the logical geographic camera. Input from either
+ * eye is routed through a controller for that screen region and updates one shared view state.
+ */
+export class WebXRPresentation {
+  /** Logical view shared by every eye. */
+  declare readonly view: XRDeckView;
+  /** Shared geographic state, updated by either controller. */
+  declare viewState: XRViewState;
+  /** Content pose in the independent room reference space. */
+  declare placement: XRPlacement;
+  /** Requested presentation mode. */
+  declare mode: XRPresentationMode;
+  /** Resolved mode used to size controller hit regions. */
+  declare controllerMode: XRPresentationMode;
+  /** Mono or left-eye controller. */
+  declare controller: XRDeckController | null;
+  /** Right-eye controller, present only during stereo preview. */
+  declare rightController: XRDeckController | null;
+  /** Shared desktop input manager. */
+  declare eventManager: EventManager | null;
+  /** Last rendered canvas dimensions. */
+  declare controllerSize: XRViewportSize;
+  /** Callbacks installed before either controller is created. */
+  declare controllerCallbacks: Required<Omit<WebXRControllerOptions, 'element'>>;
+
+  /** Initialize one logical view without allocating controllers or GPU resources. */
+  constructor({view, viewState, placement, mode = 'auto'}: WebXRPresentationOptions) {
+    this.view = view;
+    this.viewState = {...viewState};
+    this.placement = placement || createDefaultPlacement(view, viewState);
+    this.mode = mode;
+    this.controllerMode = mode;
+    this.controller = null;
+    this.rightController = null;
+    this.eventManager = null;
+    this.controllerSize = {width: 1, height: 1};
+  }
+
+  /** Read the shared state used by mono and both stereo eyes. */
+  getViewState() {
+    return this.viewState;
+  }
+
+  /** Apply a logical navigation patch and synchronize desktop controllers. */
+  setViewState(update: Partial<XRViewState> | ((state: XRViewState) => Partial<XRViewState>)) {
+    const patch = typeof update === 'function' ? update(this.viewState) : update;
+    this.viewState = {...this.viewState, ...patch};
+    this.updateController(this.controllerSize);
+    return this.viewState;
+  }
+
+  /** Replace the room placement without changing geographic navigation. */
+  setPlacement(placement: XRPlacement) {
+    this.placement = placement;
+  }
+
+  /** Change the requested presentation mode and desktop hit regions. */
+  setMode(mode: XRPresentationMode) {
+    this.mode = mode;
+    this.updateController(this.controllerSize);
+  }
+
+  /** Install deck.gl gestures backed by the shared logical navigation state. */
+  attachController({element, timeline, onViewStateChange = () => {}, onStateChange = () => {}}: WebXRControllerOptions) {
+    const controllerOptions = this.view.controller;
+    if (!controllerOptions) return null;
+    // Match deck.gl's gesture setup: mjolnir does not register recognizers by default.
+    // Order matters because gesture dependencies refer to previously registered names.
+    this.eventManager = new EventManager(element, {
+      touchAction: 'none',
+      recognizers: [
+        {recognizer: new Pan({event: 'multipan', threshold: 10, pointers: 2, trackpad: true})},
+        {recognizer: new Pinch({event: 'pinch', trackpad: true}), requireFailure: ['multipan']},
+        {recognizer: new Pan({event: 'pan', threshold: 1}),
+          recognizeWith: ['pinch'], requireFailure: ['multipan']},
+        {recognizer: new Tap({event: 'dblclick', taps: 2, enable: false})},
+        {recognizer: new DoubleClickDrag({event: 'dblclickdrag', enable: false}),
+          recognizeWith: ['dblclick']},
+        {recognizer: new Tap({event: 'click'}), recognizeWith: ['dblclickdrag'],
+          requireFailure: ['dblclick', 'dblclickdrag']}
+      ]
+    });
+    this.controllerCallbacks = {timeline, onViewStateChange, onStateChange};
+    this.controller = this.createController();
+    this.updateController(this.controllerSize);
+    return this.controller;
+  }
+
+  /** Create a controller for one screen region backed by the shared view state. */
+  createController() {
+    const controllerOptions = this.view.controller;
+    if (!controllerOptions || !this.eventManager) {
+      throw new Error('Attach a controller before creating an eye controller');
+    }
+    const {timeline, onViewStateChange, onStateChange} = this.controllerCallbacks;
+    const Controller = controllerOptions.type;
+    return new Controller({
+      timeline,
+      eventManager: this.eventManager,
+      makeViewport: (viewState: XRViewState) => requireViewport(this.view.makeViewport({
+        ...this.controllerSize,
+        width: this.controllerMode === 'stereo-preview'
+          ? this.controllerSize.width / 2 : this.controllerSize.width,
+        viewState
+      })),
+      onViewStateChange: (parameters: ViewStateChangeParameters<XRViewState>) => {
+        this.viewState = {...parameters.viewState};
+        this.updateController(this.controllerSize);
+        onViewStateChange({...parameters, viewState: this.viewState});
+      },
+      onStateChange
+    });
+  }
+
+  /** Match controller rectangles to the current mono or stereo dimensions. */
+  updateController({width, height}: XRViewportSize, mode = this.mode) {
+    this.controllerSize = {width, height};
+    this.controllerMode = mode;
+    if (!this.controller) return;
+    if (mode === 'stereo-preview') {
+      this.rightController ||= this.createController();
+      width /= 2;
+    } else if (this.rightController) {
+      this.rightController.finalize();
+      this.rightController = null;
+    }
+    const viewport = this.view.makeViewport({width, height, viewState: this.viewState});
+    const controllerOptions = this.view.controller;
+    if (!viewport || !controllerOptions) return;
+    this.controller.setProps({
+      ...this.viewState,
+      ...controllerOptions,
+      id: this.view.id,
+      x: viewport.x,
+      y: viewport.y,
+      width: viewport.width,
+      height: viewport.height
+    });
+    this.rightController?.setProps({
+      ...this.viewState,
+      ...controllerOptions,
+      // Keyboard input is global to the shared EventManager. Only the left
+      // controller handles it, otherwise FirstPersonView moves twice per key.
+      keyboard: false,
+      id: `${this.view.id}-right`,
+      x: width,
+      y: viewport.y,
+      width: viewport.width,
+      height: viewport.height
+    });
+  }
+
+  /** Advance desktop zoom and inertia transitions once per animation frame. */
+  updateTransitions() {
+    this.controller?.updateTransition();
+    this.rightController?.updateTransition();
+  }
+
+  /** Release desktop input resources without taking ownership of the host view. */
+  finalize() {
+    this.controller?.finalize();
+    this.rightController?.finalize();
+    this.eventManager?.destroy();
+    this.controller = null;
+    this.rightController = null;
+    this.eventManager = null;
+  }
+
+  /** Derive one eye viewport and its matching Tangram camera contract. */
+  makeRenderView({id, width, height, eyeOffset = 0, viewportX = 0}: XRViewportSize & {
+    id: string; eyeOffset?: number; viewportX?: number;
+  }): XRPresentationRenderView & {hostFrame: XRHostFrameFields} {
+    const deckViewport = this.view.makeEyeViewport({
+      width,
+      height,
+      viewState: this.viewState,
+      eyeOffset
+    });
+    if (!deckViewport) {
+      throw new Error(`${this.view.constructor.displayName} produced an empty viewport`);
+    }
+    const hostFrame = this.view.getHostFrame(deckViewport);
+    return {
+      id,
+      viewport: {x: viewportX, y: 0, width, height},
+      camera: hostFrame.camera,
+      hostFrame,
+      deckViewport
+    };
+  }
+
+  /** Derive parallel off-axis desktop eyes with one shared geographic anchor. */
+  makeStereoRenderViews({width, height, interpupillaryDistance = DEFAULT_INTERPUPILLARY_DISTANCE}: XRViewportSize & {
+    interpupillaryDistance?: number;
+  }): XRPresentationRenderView[] {
+    // Translate along camera-right, not geographic east or globe longitude.
+    // The off-axis projections keep the geographic anchor on the same screen
+    // position in both eyes, while nearer/farther geometry gains stereo depth.
+    return ['left-eye', 'right-eye'].map((id, index) => {
+      const renderView = this.makeRenderView({id, width, height});
+      const viewport = renderView.deckViewport;
+      const viewScale = Math.hypot(
+        viewport.viewMatrix[0], viewport.viewMatrix[4], viewport.viewMatrix[8]
+      );
+      const unitsPerXR = this.placement.type === 'globe'
+        ? viewScale * 256 / this.placement.radius
+        : viewScale * viewport.distanceScales.unitsPerMeter[2] *
+          (this.placement.type === 'map' ? this.placement.metersPerXRUnit : 1);
+      const eyeOffset = (index === 0 ? -0.5 : 0.5) * interpupillaryDistance * unitsPerXR;
+      const anchor = viewport.projectPosition([
+        this.viewState.longitude, this.viewState.latitude, 0
+      ]);
+      const anchorInView = new Matrix4().copy(viewport.viewMatrix).transformAsPoint(anchor);
+      // At/above the first-person horizon the ground anchor is not in front of
+      // the eye. Keep the convergence plane 100 physical meters forward instead
+      // of collapsing it to 1 cm and shearing both frusta away from ground.
+      const anchorDistance = -anchorInView[2];
+      const convergence = anchorDistance > 0.01 ? anchorDistance
+        : this.placement.type === 'first-person' ? 100 * unitsPerXR : 0.01;
+      const viewMatrix = new Matrix4().translate([-eyeOffset, 0, 0])
+        .multiplyRight(renderView.camera.view);
+      const projectionMatrix = new Matrix4().copy(viewport.projectionMatrix);
+      projectionMatrix[8] -= projectionMatrix[0] * eyeOffset / convergence;
+      renderView.camera = {
+        ...renderView.camera,
+        view: viewMatrix,
+        projection: this.view.getXRProjectionMatrix({projectionMatrix, viewMatrix}),
+        position: this.placement.type === 'globe'
+          ? getCameraPosition(viewMatrix)
+          : renderView.camera.position
+      };
+      renderView.hostFrame = {...renderView.hostFrame, camera: renderView.camera};
+      if (typeof this.view.getHostFrameForCamera === 'function') {
+        renderView.hostFrame = this.view.getHostFrameForCamera(viewport, renderView.camera);
+      }
+      return renderView;
+    });
+  }
+
+  /** Compose native XR eye matrices with content placement. */
+  makeXRRenderViews({frameState, placementMatrix}: {frameState: XRFrameState; placementMatrix: Matrix4}): XRPresentationRenderView[] {
+    const logicalViewport = requireViewport(this.view.makeViewport({
+      width: Math.max(...frameState.views.map((view) => view.viewport[0] + view.viewport[2])),
+      height: Math.max(...frameState.views.map((view) => view.viewport[1] + view.viewport[3])),
+      viewState: this.viewState
+    }));
+    return frameState.views.map((xrView) => {
+      const [x, y, width, height] = xrView.viewport;
+      const viewMatrix = new Matrix4().copy(xrView.viewMatrix).multiplyRight(placementMatrix);
+      const projectionMatrix = new Matrix4().copy(xrView.projectionMatrix);
+      const renderView: XRPresentationRenderView = {
+        id: xrView.eye || `eye-${xrView.index}`,
+        viewport: {x, y, width, height},
+        camera: {
+          view: viewMatrix,
+          projection: this.view.getXRProjectionMatrix({projectionMatrix, viewMatrix}),
+          position: this.placement.type === 'globe'
+            ? getCameraPosition(viewMatrix)
+            : [0, 0, 0]
+        },
+        deckViewport: logicalViewport,
+        xrView,
+        view: this.view,
+        viewState: this.viewState
+      };
+      if (typeof this.view.getHostFrameForCamera === 'function') {
+        renderView.hostFrame = this.view.getHostFrameForCamera(logicalViewport, renderView.camera, {width, height});
+      }
+      return renderView;
+    });
+  }
+
+  /** Prepare a presentation snapshot and renderer-independent host frame. */
+  createFrame({width, height, frameState, mode = this.mode, interpupillaryDistance, tileResources}: WebXRFrameOptions): XRPresentationFrame {
+    const resolvedMode = resolveMode(mode, frameState);
+    // A caller may select a presentation mode per frame without calling setMode.
+    // Keep controller hit regions and gesture viewport dimensions in lockstep.
+    this.updateController({width, height}, resolvedMode);
+    const logicalViewport = this.view.makeViewport({width, height, viewState: this.viewState});
+    if (!logicalViewport) throw new Error('The logical deck.gl view produced an empty viewport');
+    let renderViews;
+    if (resolvedMode === 'immersive-vr' && frameState?.views?.length) {
+      const placementMatrix = createXRPlacementMatrix(this.placement, this.viewState);
+      renderViews = this.makeXRRenderViews({frameState, placementMatrix});
+    } else if (resolvedMode === 'stereo-preview') {
+      const eyeWidth = Math.floor(width / 2);
+      renderViews = this.makeStereoRenderViews({
+        width: eyeWidth,
+        height,
+        interpupillaryDistance
+      });
+      renderViews[1].viewport.x = eyeWidth;
+    } else {
+      renderViews = [this.makeRenderView({id: 'mono', width, height})];
+    }
+    return {
+      mode: resolvedMode,
+      logicalViewport,
+      renderViews,
+      hostFrame: {...this.createHostFrame({width, height, renderViews, frameState}), tileResources}
+    };
+  }
+
+  /** Union per-eye visibility while preserving camera-independent host metadata. */
+  createHostFrame({width, height, renderViews, frameState}: XRViewportSize & {
+    renderViews: XRPresentationRenderView[]; frameState?: XRFrameState;
+  }): HostFrameOptions {
+    const fields = renderViews.find((renderView) => renderView.hostFrame)?.hostFrame;
+    const fallback = this.view.getHostFrame(
+      requireViewport(this.view.makeViewport({width, height, viewState: this.viewState}))
+    );
+    const frameFields = fields || fallback;
+    let projection = frameFields.projection;
+    if (this.placement.type === 'globe' && frameState?.views?.length) {
+      projection = {
+        ...frameFields.projection,
+        type: 'globe',
+        visibleBounds: getXRGlobeVisibleBounds({
+          views: frameState.views,
+          placement: this.placement,
+          viewState: this.viewState
+        })
+      };
+    }
+    return {
+      viewport: {x: 0, y: 0, width, height},
+      geographicAnchor: frameFields.view,
+      projection,
+      renderViews: renderViews.map(renderView => ({
+        ...renderView,
+        // Keep native render-pass rectangles untouched, but HostFrame routes
+        // top-origin canvas pixels (desktop CSS, immersive framebuffer units).
+        viewport: renderView.xrView ? {...renderView.viewport,
+          y: height - (renderView.viewport.y ?? 0) - renderView.viewport.height} : renderView.viewport,
+        geographicAnchor: renderView.hostFrame?.view,
+        // Immersive globe bounds already describe the union in room space.
+        projection: frameState?.views?.length && this.placement.type === 'globe'
+          ? undefined : renderView.hostFrame?.projection
+      })),
+      activeRenderViewId: renderViews[0].id,
+      tileBuffer: frameFields.tileBuffer,
+      globePreloadZoom: frameFields.globePreloadZoom
+    };
+  }
+
+  /** Apply navigation intents; non-navigation signals remain owned by the caller. */
+  dispatchInteractionIntent(intent: XRInteractionIntent) {
+    if (intent.type !== 'navigate') return this.viewState;
+    const [horizontal = 0, vertical = 0] = intent.delta;
+    if (intent.action === 'turn' || intent.action === 'rotate') {
+      return this.setViewState({bearing: (this.viewState.bearing || 0) + horizontal});
+    }
+    if (intent.action === 'pitch') {
+      return this.setViewState({pitch: (this.viewState.pitch || 0) + vertical});
+    }
+    if (intent.action === 'zoom') {
+      return this.setViewState({zoom: (this.viewState.zoom || 0) + vertical});
+    }
+    if (this.placement.type === 'first-person') {
+      const position = this.placement.position || [0, 0, 0];
+      this.placement = {
+        ...this.placement,
+        position: [position[0] + horizontal, position[1] + vertical, position[2]]
+      };
+      return this.viewState;
+    }
+    return this.setViewState({
+      longitude: (this.viewState.longitude || 0) + horizontal,
+      latitude: (this.viewState.latitude || 0) + vertical
+    });
+  }
+}
+
+/** Backward-compatible name used by the first WebXR example iteration. */
+export class WebXRViewManager extends WebXRPresentation {}
+
+/** Resolve unavailable immersive modes to explicit stereo preview. */
+function resolveMode(mode: XRPresentationMode, frameState?: XRFrameState): Exclude<XRPresentationMode, 'auto'> {
+  if (mode === 'auto') return frameState?.views?.length ? 'immersive-vr' : 'mono';
+  if (mode === 'immersive-vr' && !frameState?.views?.length) return 'stereo-preview';
+  return mode;
+}
+
+/** Choose the existing room presentation defaults for each geographic view. */
+function createDefaultPlacement(view: XRDeckView, viewState: XRViewState): XRPlacement {
+  const anchor: XRVector3 = [viewState.longitude || 0, viewState.latitude || 0, 0];
+  if (view.constructor.displayName === 'WebXRGlobeView') {
+    return {type: 'globe', anchor, pose: {position: [0, 1.35, -2.35]}, radius: 0.72};
+  }
+  if (view.constructor.displayName === 'WebXRFirstPersonView') {
+    return {type: 'first-person', origin: anchor, position: [0, 0, 0], bearing: 0};
+  }
+  return {
+    type: 'map',
+    anchor,
+    pose: {position: [0, 0.72, -1.8]},
+    metersPerXRUnit: 2500,
+    surface: {type: 'unbounded'}
+  };
+}
+
+/** Preserve the fixed-size camera position required by the renderer contract. */
+function getCameraPosition(matrix: Matrix4): [number, number, number] {
+  const position: [number, number, number] = [0, 0, 0];
+  new Matrix4(matrix).invert().transformAsPoint([0, 0, 0], position);
+  return position;
+}
+
+/** Reject a hidden logical view before using its camera or controller dimensions. */
+function requireViewport(viewport: import('./types.js').XRDeckViewport | null) {
+  if (!viewport) throw new Error('The logical deck.gl view produced an empty viewport');
+  return viewport;
+}