@shopware-ag/dive 1.16.25 → 1.16.26-beta.1

package/src/ar/webxr/controller/WebXRController.ts

@@ -0,0 +1,334 @@
+import {
+    Matrix4,
+    Mesh,
+    Object3D,
+    Quaternion,
+    Vector3,
+    WebXRArrayCamera,
+} from 'three';
+import { DIVERenderer } from '../../../renderer/Renderer';
+import { DIVEScene } from '../../../scene/Scene';
+import { DIVEWebXRCrosshair } from '../crosshair/WebXRCrosshair';
+import { DIVEWebXRRaycaster } from '../raycaster/WebXRRaycaster';
+import { DIVEWebXROrigin } from '../origin/WebXROrigin';
+import {
+    DIVETouchscreenEvents,
+    DIVEWebXRTouchscreenControls,
+} from '../touchscreencontrols/WebXRTouchscreenControls';
+import { type DIVEMovable } from '../.././../interface/Movable';
+import { findInterface } from '../../../helper/findInterface/findInterface';
+
+export class DIVEWebXRController extends Object3D {
+    // general members
+    private _renderer: DIVERenderer;
+    private _scene: DIVEScene;
+    private _session: XRSession;
+
+    private _frameBuffer: XRFrame | null = null;
+
+    // raycaster members
+    private _xrRaycaster: DIVEWebXRRaycaster;
+    private _origin: DIVEWebXROrigin;
+
+    // crosshair
+    private _crosshair: DIVEWebXRCrosshair;
+
+    // controller members
+    private _touchscreenControls: DIVEWebXRTouchscreenControls;
+    private _handNodeInitialPosition = new Vector3();
+    private _xrCamera: WebXRArrayCamera;
+    private _placed: boolean = false;
+
+    // grabbing
+    private _grabbedObject: Object3D | null = null;
+    private _arHitPosition: Vector3 = new Vector3();
+    private _arHitQuaternion: Quaternion = new Quaternion();
+    private _arHitScale: Vector3 = new Vector3(1, 1, 1);
+
+    // grabbing position
+    private _initialObjectPosition: Vector3 | null = null;
+    private _initialRaycastHit: Vector3 | null = null;
+    private _deltaRaycastHit: Vector3 = new Vector3();
+
+    // grabbing rotation
+    private _touchQuaterion: Quaternion = new Quaternion();
+
+    // grabbing scale
+    private _touchScale: number = 1;
+    private _scaleThreshold: number = 0.1;
+
+    constructor(session: XRSession, renderer: DIVERenderer, scene: DIVEScene) {
+        super();
+
+        this._renderer = renderer;
+        this._scene = scene;
+        this._session = session;
+
+        this._xrRaycaster = new DIVEWebXRRaycaster(session, renderer, scene);
+        this._origin = new DIVEWebXROrigin(this._session, this._renderer, [
+            'plane',
+        ]);
+
+        this._crosshair = new DIVEWebXRCrosshair();
+        this._crosshair.visible = false;
+
+        this._xrCamera = this._renderer.xr.getCamera();
+
+        this._scene.XRRoot.XRHandNode.position.set(0, -0.05, -0.25);
+        this._handNodeInitialPosition =
+            this._scene.XRRoot.XRHandNode.position.clone();
+
+        this._touchscreenControls = new DIVEWebXRTouchscreenControls(
+            this._session,
+        );
+
+        // translating
+        this._touchscreenControls.Subscribe('TOUCH_START', () =>
+            this.onTouchStart(),
+        );
+        this._touchscreenControls.Subscribe('TOUCH_MOVE', () =>
+            this.onTouchMove(),
+        );
+        this._touchscreenControls.Subscribe('TOUCH_END', (p) =>
+            this.onTouchEnd(p),
+        );
+
+        // rotating
+        this._touchscreenControls.Subscribe('ROTATE_START', () =>
+            this.onRotateStart(),
+        );
+        this._touchscreenControls.Subscribe('ROTATE_MOVE', (p) =>
+            this.onRotateMove(p),
+        );
+
+        // scaling
+        this._touchscreenControls.Subscribe('PINCH_START', () =>
+            this.onPinchStart(),
+        );
+        this._touchscreenControls.Subscribe('PINCH_MOVE', (p) =>
+            this.onPinchMove(p),
+        );
+    }
+
+    public async Init(): Promise<this> {
+        this.prepareScene();
+
+        await this.initOrigin();
+        await this.initRaycaster();
+
+        return Promise.resolve(this);
+    }
+
+    public Dispose(): void {
+        this.restoreScene();
+
+        this._origin.Dispose();
+        this._xrRaycaster.Dispose();
+
+        // reset placement members
+        this._placed = false;
+    }
+
+    public Update(frame: XRFrame): void {
+        this._frameBuffer = frame;
+
+        if (!this._placed) {
+            this._xrCamera.updateMatrixWorld();
+            this._scene.XRRoot.XRHandNode.position.copy(
+                this._handNodeInitialPosition
+                    .clone()
+                    .applyMatrix4(this._xrCamera.matrixWorld),
+            );
+            this._scene.XRRoot.XRHandNode.quaternion.setFromRotationMatrix(
+                this._xrCamera.matrixWorld,
+            );
+
+            if (this._origin) {
+                this._origin.Update(frame);
+            }
+        }
+    }
+
+    // placement
+    private async initOrigin(): Promise<void> {
+        // initialize origin
+        this._origin = await this._origin.Init();
+
+        // set resolve callback: place objects at origin when it is set
+        this._origin.originSet.then(() => {
+            this.placeObjects(this._origin.matrix);
+        });
+    }
+
+    private placeObjects(matrix: Matrix4): void {
+        this._scene.XRRoot.XRModelRoot.matrix.copy(matrix);
+        [...this._scene.XRRoot.XRHandNode.children].forEach((child) => {
+            this._scene.XRRoot.XRModelRoot.add(child);
+        });
+        this._placed = true;
+    }
+
+    // grabbing
+    private updateObject(): void {
+        if (!this._grabbedObject) return;
+
+        this._grabbedObject.position.copy(this._arHitPosition);
+        this._grabbedObject.quaternion.copy(
+            this._arHitQuaternion.clone().multiply(this._touchQuaterion),
+        );
+        this._grabbedObject.scale.copy(
+            new Vector3(
+                this._touchScale,
+                this._touchScale,
+                this._touchScale,
+            ).multiply(this._arHitScale),
+        );
+    }
+
+    private onTouchStart(): void {
+        const sceneHits = this._xrRaycaster.GetSceneIntersections();
+        console.log('sceneHits', sceneHits);
+        if (sceneHits.length === 0) return;
+        if (!sceneHits[0].object) return;
+
+        const moveable = findInterface<DIVEMovable>(
+            sceneHits[0].object,
+            'isMovable',
+        );
+        if (!moveable) return;
+
+        this._grabbedObject = moveable;
+    }
+
+    private onTouchMove(): void {
+        // raycast ar
+        if (!this._frameBuffer) return;
+        if (!this._grabbedObject) return;
+
+        const intersections = this._xrRaycaster.GetARIntersections(
+            this._frameBuffer,
+        );
+        if (intersections.length === 0) {
+            this._crosshair.visible = false;
+            return;
+        }
+
+        const hit = intersections[0];
+
+        this._crosshair.visible = true;
+        this._crosshair.matrix.copy(hit.matrix);
+
+        if (!this._grabbedObject) return;
+
+        // if initial values have been reset by TOUCH_END event then set them again
+        if (!this._initialObjectPosition || !this._initialRaycastHit) {
+            this._initialObjectPosition = this._grabbedObject.position.clone();
+            this._initialRaycastHit = hit.point.clone();
+        }
+
+        // decompose hit matrix to apply hit matrix to object
+        hit.matrix.decompose(
+            this._arHitPosition,
+            this._arHitQuaternion,
+            this._arHitScale,
+        );
+
+        // calculate raycast hit delta
+        this._deltaRaycastHit.copy(
+            hit.point.clone().sub(this._initialRaycastHit),
+        );
+
+        // apply moved raycast delta to actual object position
+        this._arHitPosition.copy(
+            this._initialObjectPosition.clone().add(this._deltaRaycastHit),
+        );
+
+        console.log('arHitPosition', this._arHitPosition);
+
+        this.updateObject();
+    }
+
+    private onTouchEnd(payload: DIVETouchscreenEvents['TOUCH_END']): void {
+        if (payload.touchCount === 0) {
+            this._crosshair.visible = false;
+
+            // reset grab
+            this._initialObjectPosition = null;
+            this._initialRaycastHit = null;
+            this._grabbedObject = null;
+        }
+    }
+
+    private _startTouchQuaternion: Quaternion = new Quaternion();
+    private onRotateStart(): void {
+        this._startTouchQuaternion = this._touchQuaterion.clone();
+    }
+
+    private onRotateMove(payload: DIVETouchscreenEvents['ROTATE_MOVE']): void {
+        this._touchQuaterion.setFromAxisAngle(
+            new Vector3(0, -1, 0),
+            payload.delta * 3,
+        );
+        this._touchQuaterion.multiply(this._startTouchQuaternion);
+        this.updateObject();
+    }
+
+    private _startTouchScale: number = 1;
+    private onPinchStart(): void {
+        this._startTouchScale = this._touchScale;
+    }
+
+    private onPinchMove(payload: DIVETouchscreenEvents['PINCH_MOVE']): void {
+        this._touchScale = this._startTouchScale * payload.current;
+        this.updateObject();
+    }
+
+    // prepare & cleanup scene
+    private prepareScene(): void {
+        this._scene.XRRoot.XRModelRoot.matrixAutoUpdate = false;
+
+        // initialize crosshair
+        this._scene.add(this._crosshair);
+
+        // hang current scene children to hang node
+        const children: Object3D[] = [];
+        this._scene.Root.children.forEach((child) => {
+            const clone = child.clone();
+            clone.layers.enableAll();
+            clone.traverse((obj) => {
+                obj.layers.enableAll();
+                if (obj instanceof Mesh) {
+                    obj.scale.set(0.1, 0.1, 0.1);
+                }
+            });
+            clone.position.set(0, 0, 0);
+            children.push(clone);
+        });
+        this._scene.XRRoot.XRHandNode.add(...children);
+    }
+
+    private restoreScene(): void {
+        this._scene.remove(this._crosshair);
+
+        // clear hang node and remove attached models
+        this._scene.XRRoot.XRHandNode.clear();
+        this._scene.XRRoot.XRModelRoot.clear();
+
+        this._scene.XRRoot.XRModelRoot.matrixAutoUpdate = true;
+    }
+
+    // raycast
+    private async initRaycaster(): Promise<void> {
+        // initialize raycaster
+        await this._xrRaycaster.Init();
+
+        // check if successful
+        if (!this._xrRaycaster) {
+            console.error(
+                'Raycaster not initialized successfully. Aborting WebXR...',
+            );
+            this.Dispose();
+            return Promise.reject();
+        }
+    }
+}

package/src/ar/webxr/crosshair/WebXRCrosshair.ts

@@ -0,0 +1,35 @@
+import { Mesh, MeshBasicMaterial, Object3D, RingGeometry } from 'three';
+
+export class DIVEWebXRCrosshair extends Object3D {
+    public set mesh(mesh: Mesh | undefined) {
+        this.clear();
+
+        if (mesh) {
+            this.add(mesh);
+        }
+    }
+
+    constructor(mesh?: Mesh) {
+        super();
+
+        if (mesh) {
+            this.mesh = mesh;
+        } else {
+            this.UseDefaultMesh();
+        }
+
+        this.matrixAutoUpdate = false;
+
+        return this;
+    }
+
+    public UseDefaultMesh(): void {
+        const geometry = new RingGeometry(0.08, 0.1, 32).rotateX(-Math.PI / 2);
+        const material = new MeshBasicMaterial();
+        this.mesh = new Mesh(geometry, material);
+    }
+
+    public UpdateFromPose(pose: XRPose): void {
+        this.matrix.fromArray(pose.transform.matrix);
+    }
+}

package/src/ar/webxr/origin/WebXROrigin.ts

@@ -0,0 +1,191 @@
+import { Matrix4, Quaternion, Vector3 } from 'three';
+import { DIVERenderer } from '../../../renderer/Renderer';
+
+export class DIVEWebXROrigin {
+    private _renderer: DIVERenderer;
+    private _session: XRSession;
+
+    private _requesting: boolean;
+    private _initialized: boolean;
+
+    private _referenceSpaceBuffer: XRReferenceSpace | null;
+    private _hitTestSource: XRHitTestSource | null;
+
+    private _entityTypes: XRHitTestTrackableType[];
+
+    private _hitTestResultBuffer: XRHitTestResult[];
+    private _raycastHitCounter: number = 0;
+
+    private _originSet: Promise<void>;
+    private _originSetResolve: (value: void) => void = () => {};
+    public get originSet(): Promise<void> {
+        return this._originSet;
+    }
+
+    private _matrix: Matrix4;
+    public get matrix(): Matrix4 {
+        return this._matrix;
+    }
+    private set matrix(value: Matrix4) {
+        this._matrix = value;
+        this._matrix.decompose(this._position, this._quaternion, this._scale);
+    }
+
+    private _position: Vector3;
+    public get position(): Vector3 {
+        return this._position;
+    }
+
+    private _quaternion: Quaternion;
+    public get quaternion(): Quaternion {
+        return this._quaternion;
+    }
+
+    private _scale: Vector3;
+    public get scale(): Vector3 {
+        return this._scale;
+    }
+
+    constructor(
+        session: XRSession,
+        renderer: DIVERenderer,
+        entityTypes?: XRHitTestTrackableType[],
+    ) {
+        this._renderer = renderer;
+        this._session = session;
+
+        this._originSet = new Promise<void>((resolve) => {
+            this._originSetResolve = resolve;
+        });
+
+        this._requesting = false;
+        this._initialized = false;
+
+        this._referenceSpaceBuffer = null;
+        this._hitTestSource = null;
+
+        this._entityTypes = entityTypes || ['plane'];
+
+        this._hitTestResultBuffer = [];
+
+        // set up promises and executors
+        this._matrix = new Matrix4();
+        this._position = new Vector3();
+        this._quaternion = new Quaternion();
+        this._scale = new Vector3();
+
+        // when origin is set, decompose matrix into position, quaternion and scale
+        this._originSet.then(() => {
+            // decompose matrix into position, quaternion and scale
+            this._matrix.decompose(
+                this._position,
+                this._quaternion,
+                this._scale,
+            );
+        });
+    }
+
+    public async Init(): Promise<this> {
+        if (this._initialized) {
+            return Promise.resolve(this);
+        }
+
+        if (!this._session) {
+            console.error(
+                'DIVEWebXROrigin: No session set in Init()! Aborting initialization...',
+            );
+            return Promise.reject();
+        }
+
+        if (this._requesting) {
+            console.error(
+                'DIVEWebXROrigin: Currently initializing! Aborting initialization...',
+            );
+            return Promise.reject();
+        }
+
+        this._requesting = true;
+        const referenceSpace =
+            await this._session.requestReferenceSpace('viewer');
+        this._hitTestSource =
+            (await this._session.requestHitTestSource!({
+                space: referenceSpace,
+                entityTypes: this._entityTypes,
+            })) || null;
+        this._requesting = false;
+
+        if (!this._hitTestSource) {
+            return Promise.reject();
+        }
+
+        this._initialized = true;
+
+        return Promise.resolve(this);
+    }
+
+    public Dispose(): void {
+        this._initialized = false;
+        this._requesting = false;
+
+        this._hitTestSource?.cancel();
+
+        this._hitTestSource = null;
+        this._hitTestResultBuffer = [];
+
+        this._matrix = new Matrix4();
+        this._position = new Vector3();
+        this._quaternion = new Quaternion();
+        this._scale = new Vector3();
+    }
+
+    public Update(frame: XRFrame): void {
+        if (!this._initialized) return;
+
+        if (!this._hitTestSource) {
+            throw new Error(
+                'DIVEWebXRRaycaster: Critical Error: HitTestSource not available but WebXROrigin is initialized!',
+            );
+        }
+
+        // get hit test results
+        this._hitTestResultBuffer = frame.getHitTestResults(
+            this._hitTestSource,
+        );
+        if (this._hitTestResultBuffer.length > 0) {
+            // hit found
+            this._referenceSpaceBuffer = this._renderer.xr.getReferenceSpace();
+
+            // if there is no reference space, hit will be counted as lost for this frame
+            if (!this._referenceSpaceBuffer) {
+                this.onHitLost();
+                return;
+            }
+
+            const pose = this._hitTestResultBuffer[0].getPose(
+                this._referenceSpaceBuffer,
+            );
+            if (!pose) {
+                this.onHitLost();
+                return;
+            }
+
+            this.onHitFound(pose);
+        } else {
+            this.onHitLost();
+        }
+    }
+
+    private onHitFound(pose: XRPose): void {
+        this._raycastHitCounter++;
+
+        this.matrix.fromArray(pose.transform.matrix);
+
+        if (this._raycastHitCounter > 50) {
+            this._originSetResolve();
+        }
+    }
+
+    private onHitLost(): void {
+        this._raycastHitCounter = 0;
+    }
+}

package/src/ar/webxr/overlay/Overlay.ts

@@ -0,0 +1,50 @@
+export class Overlay {
+    private _element: HTMLDivElement;
+    public get Element(): HTMLDivElement {
+        return this._element;
+    }
+
+    private _closeButton: SVGSVGElement;
+    public get CloseButton(): SVGSVGElement {
+        return this._closeButton;
+    }
+
+    constructor() {
+        // create div
+        this._element = document.createElement('div');
+
+        // create and append close button to overlay
+        this._closeButton = this.createCloseButton();
+        this._element.appendChild(this._closeButton);
+
+        // append overlay to body
+        document.body.appendChild(this._element);
+    }
+
+    private createCloseButton(): SVGSVGElement {
+        // create svg path
+        const path = document.createElementNS(
+            'http://www.w3.org/2000/svg',
+            'path',
+        );
+        path.setAttribute('d', 'M 12,12 L 28,28 M 28,12 12,28');
+        path.setAttribute('stroke', '#fff');
+        path.setAttribute('stroke-width', '2');
+
+        // create svg
+        const svg = document.createElementNS(
+            'http://www.w3.org/2000/svg',
+            'svg',
+        );
+        svg.setAttribute('width', '38');
+        svg.setAttribute('height', '38');
+        svg.style.position = 'absolute';
+        svg.style.right = '20px';
+        svg.style.top = '20px';
+
+        // append path to svg
+        svg.appendChild(path);
+
+        return svg;
+    }
+}