@needle-tools/engine 3.28.7-beta → 3.28.7-beta.1

package/src/engine-components/webxr/WebARSessionRoot.ts

@@ -1,446 +1,446 @@
-import { Behaviour, GameObject } from "../Component.js";
-import { Matrix4, Object3D, Plane, Quaternion, Ray, Raycaster, Vector2, Vector3 } from "three";
-import { WebAR, WebXR } from "./WebXR.js";
-import { InstancingUtil } from "../../engine/engine_instancing.js";
-import { serializable } from "../../engine/engine_serialization_decorator.js";
-import { Context } from "../../engine/engine_context.js";
-import { isQuest } from "../../engine/engine_utils.js";
-
-// https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
-
-const tempMatrix = new Matrix4();
-
-export class WebARSessionRoot extends Behaviour {
-
-    webAR: WebAR | null = null;
-
-    get rig(): Object3D | undefined {
-        return this.webAR?.webxr.Rig;
-    }
-
-    @serializable()
-    invertForward: boolean = false;
-
-    /** Preview feature: enable touch transform */
-    @serializable()
-    arTouchTransform: boolean = false;
-
-    @serializable()
-    get arScale(): number {
-        return this._arScale;
-    }
-    set arScale(val: number) {
-        if (val === this._arScale) return;
-        this._arScale = val;
-        this.setScale(val);
-    }
-
-    private readonly _initalMatrix = new Matrix4();
-    private readonly _selectStartFn = this.onSelectStart.bind(this);
-    private readonly _selectEndFn = this.onSelectEnd.bind(this);
-    private userInput?: WebXRSessionRootUserInput;
-
-    start() {
-        const xr = GameObject.findObjectOfType(WebXR);
-        if (xr) {
-            xr.Rig.updateMatrix();
-            this._initalMatrix.copy(xr.Rig.matrix);
-        }
-    }
-
-    private _arScale: number = 1;
-    private _rig: Object3D | null = null;
-    private _startPose: Matrix4 | null = null;
-    private _placementPose: Matrix4 | null = null;
-    private _isTouching: boolean = false;
-    private _rigStartPose: Matrix4 | undefined | null = null;
-    private _gotFirstHitTestResult: boolean = false;
-    private _anchor: XRAnchor | null = null;
-
-    onBegin(session: XRSession) {
-
-        this._placementPose = null;
-        this.gameObject.visible = false;
-        this.gameObject.matrixAutoUpdate = false;
-        this._startPose = this.gameObject.matrix.clone();
-        this._rigStartPose = this.rig?.matrix.clone();
-        this._gotFirstHitTestResult = false;
-        this._anchor = null;
-        session.addEventListener('selectstart', this._selectStartFn);
-        session.addEventListener('selectend', this._selectEndFn);
-        // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
-
-        // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
-        this.gameObject.visible = false;
-
-        if (this.rig) {
-            // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
-            // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
-            // immersive Ar right now to place the user/offset the session
-            this.rig.matrixAutoUpdate = true;
-            this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
-        }
-
-        // TODO this is duplicate to WebXR events AND engine events, would be better in one place
-        this.dispatchEvent(new CustomEvent('onBeginSession'));
-    }
-
-    onUpdate(rig: Object3D | null, _session: XRSession, hit: XRHitTestResult | null, pose: XRPose | null | undefined): boolean {
-
-        if (pose && !this._placementPose) {
-            if (!this._gotFirstHitTestResult) {
-                this._gotFirstHitTestResult = true;
-                this.dispatchEvent(new CustomEvent('canPlaceSession', { detail: { pose: pose } }));
-            }
-
-            if (this._isTouching) {
-                // callbacks
-                const poseMatrix = new Matrix4().fromArray(pose.transform.matrix).invert();
-                this.dispatchEvent(new CustomEvent('placedSession', { detail: { pose, poseMatrix } }));
-
-                if (this.webAR) this.webAR.setReticleActive(false);
-                this.placeAt(rig, poseMatrix);
-                if (hit && pose && !isQuest()) // TODO anchors seem to behave differently with an XRRig
-                   this.onCreatePlacementAnchor(hit, pose);
-
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private onCreatePlacementAnchor(hit: XRHitTestResult, pose: XRPose) {
-        this._anchor = null;
-        hit?.createAnchor?.call(hit, pose.transform)?.then(anchor => {
-            if (this.context.isInAR)
-                this._anchor = anchor;
-        }).catch(ex => {
-            console.warn("Failed to create anchor", ex);
-        });
-    }
-
-    private _anchorMatrix: Matrix4 = new Matrix4();
-
-    onBeforeRender(frame: XRFrame | null): void {
-        if (frame && this._rig) {
-            if (this._anchor) {
-                const referenceSpace = this.context.renderer.xr.getReferenceSpace();
-                if (referenceSpace) {
-                    const pose = frame.getPose(this._anchor.anchorSpace, referenceSpace);
-                    if (pose) {
-                        const poseMatrix = this._anchorMatrix.fromArray(pose.transform.matrix).invert();
-                        this.placeAt(this._rig, poseMatrix);
-                        return;
-                    }
-                }
-            }
-            else if (this._placementPose) {
-                this.placeAt(this._rig, this._placementPose!);
-            }
-        }
-    }
-
-    private _invertedSessionRootMatrix: Matrix4 = new Matrix4();
-    private _invertForwardMatrix: Matrix4 = new Matrix4().makeRotationY(Math.PI);
-
-    placeAt(rig: Object3D | null, mat: Matrix4) {
-        if (!this._placementPose) this._placementPose = new Matrix4();
-        this._placementPose.copy(mat);
-
-        // apply session root offset
-        const invertedSessionRoot = this._invertedSessionRootMatrix.copy(this.gameObject.matrixWorld).invert();
-        this._placementPose.premultiply(invertedSessionRoot);
-        if (rig) {
-            if (this.invertForward) {
-                this._placementPose.premultiply(this._invertForwardMatrix);
-            }
-
-            if (!this.userInput) this.userInput = new WebXRSessionRootUserInput(this.context);
-            this.userInput.enable();
-
-            this._rig = rig;
-            this.setScale(this.arScale);
-        }
-        else this._rig = null;
-        this.gameObject.visible = true;
-    }
-
-    onEnd(rig: Object3D | null, _session: XRSession) {
-        this.userInput?.disable();
-        this.userInput?.reset();
-
-        this._placementPose = null;
-        this.gameObject.visible = false;
-        this.gameObject.matrixAutoUpdate = false;
-        this._anchor = null;
-        if (this._startPose) {
-            this.gameObject.matrix.copy(this._startPose);
-        }
-        if (rig) {
-            rig.matrixAutoUpdate = true;
-            if (this._rigStartPose) {
-                this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
-                // console.log(rig.position, rig.quaternion, rig.scale);
-            }
-        }
-        InstancingUtil.markDirty(this.gameObject, true);
-        // HACK to fix physics being not in correct place after exiting AR
-        setTimeout(() => {
-            if (!this.gameObject) return;
-            this.gameObject.matrixAutoUpdate = true;
-            this.gameObject.visible = true;
-        }, 100);
-    }
-
-
-    private onSelectStart() {
-        this._isTouching = true;
-    }
-
-    private onSelectEnd() {
-        this._isTouching = false;
-    }
-
-    private setScale(scale) {
-        const rig = this._rig;
-        if (!rig || !this._placementPose) {
-            return;
-        }
-        // Capture the rig position before the first time we move it during a session
-        if (!this._rigStartPose) {
-            this._rigStartPose = rig.matrix.clone();
-        }
-        // we apply the transform to the rig because we want to move the user's position for easy networking
-        rig.matrixAutoUpdate = false;
-        if (this.arTouchTransform && this.userInput) {
-            this.userInput.applyMatrixTo(this._placementPose);
-            // rig.matrix.premultiply(this.userInput.offset);
-        }
-        rig.matrix.multiplyMatrices(tempMatrix.makeScale(scale, scale, scale), this._placementPose);
-        rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
-        rig.updateMatrixWorld();
-    }
-}
-
-
-
-
-class WebXRSessionRootUserInput {
-    private static up = new Vector3(0, 1, 0);
-    private static zero = new Vector3(0, 0, 0);
-    private static one = new Vector3(1, 1, 1);
-
-    oneFingerDrag: boolean = true;
-    twoFingerRotate: boolean = true;
-    twoFingerScale: boolean = true;
-
-    readonly context: Context;
-    readonly offset: Matrix4;
-    readonly plane: Plane;
-
-    private _scale: number = 1;
-
-    // readonly translate: Vector3 = new Vector3();
-    // readonly rotation: Quaternion = new Quaternion();
-    // readonly scale: Vector3 = new Vector3(1, 1, 1);
-
-    constructor(context: Context) {
-        this.context = context;
-        this.offset = new Matrix4()
-        this.plane = new Plane();
-        this.plane.setFromNormalAndCoplanarPoint(WebXRSessionRootUserInput.up, WebXRSessionRootUserInput.zero);
-    }
-
-    private _enabled: boolean = false;
-    enable() {
-        if (this._enabled) return;
-        this._enabled = true;
-        window.addEventListener('touchstart', this.touchStart, { passive: false });
-        window.addEventListener('touchmove', this.touchMove, { passive: false });
-        window.addEventListener('touchend', this.touchEnd, { passive: false });
-    }
-    disable() {
-        if (!this._enabled) return;
-        this._enabled = false;
-        window.removeEventListener('touchstart', this.touchStart);
-        window.removeEventListener('touchmove', this.touchMove);
-        window.removeEventListener('touchend', this.touchEnd);
-    }
-    reset() {
-        this._scale = 1;
-        this.offset.identity();
-    }
-    applyMatrixTo(matrix: Matrix4) {
-        matrix.premultiply(this.offset);
-        // if (this._needsUpdate)
-        //     this.updateMatrix();
-        // matrix.premultiply(this._rotationMatrix);
-        // matrix.premultiply(this.offset).premultiply(this._rotationMatrix)
-    }
-
-    // private _needsUpdate: boolean = true;
-    // private _rotationMatrix: Matrix4 = new Matrix4();
-    // private updateMatrix() {
-    //     this._needsUpdate = false;
-    //     this._rotationMatrix.makeRotationFromQuaternion(this.rotation);
-    //     this.offset.compose(this.translate, new Quaternion(), this.scale);
-    //     // const rot = this._tempMatrix.makeRotationY(this.angle);
-    //     // this.translate.applyMatrix4(rot);
-    //     // this.offset.elements[12] = this.translate.x;
-    //     // this.offset.elements[13] = this.translate.y;
-    //     // this.offset.elements[14] = this.translate.z;
-    //     // this.offset.premultiply(rot);
-    //     // const s = this.scale;
-    //     // this.offset.premultiply(this._tempMatrix.makeScale(s, s, s));
-    // }
-
-    private prev: Map<number, { x: number, z: number, screenx: number, screeny: number }> = new Map();
-    private _didMultitouch: boolean = false;
-
-    private touchStart = (evt: TouchEvent) => {
-        for (let i = 0; i < evt.changedTouches.length; i++) {
-            const touch = evt.changedTouches[i];
-            if (!this.prev.has(touch.identifier))
-                this.prev.set(touch.identifier, {
-                    x: 0,
-                    z: 0,
-                    screenx: 0,
-                    screeny: 0,
-                });
-            const prev = this.prev.get(touch.identifier);
-            if (prev) {
-                const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
-                prev.x = pos.x;
-                prev.z = pos.z;
-                prev.screenx = touch.clientX;
-                prev.screeny = touch.clientY;
-            }
-        }
-    }
-    private touchEnd = (evt: TouchEvent) => {
-        if (evt.touches.length <= 0) {
-            this._didMultitouch = false;
-        }
-    }
-    private touchMove = (evt: TouchEvent) => {
-        if (evt.defaultPrevented) return;
-        
-        if (evt.touches.length === 1) {
-            // if we had multiple touches before due to e.g. pinching / rotating
-            // and stopping one of the touches, we don't want to move the scene suddenly
-            // this will be resettet when all touches stop
-            if (this._didMultitouch) {
-                return;
-            }
-            const touch = evt.touches[0];
-            const prev = this.prev.get(touch.identifier);
-            if (!prev) return;
-            const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
-            const dx = pos.x - prev.x;
-            const dy = pos.z - prev.z;
-            if (dx === 0 && dy === 0) return;
-            if (this.oneFingerDrag)
-                this.addMovement(dx, dy);
-            prev.x = pos.x;
-            prev.z = pos.z;
-            prev.screenx = touch.clientX;
-            prev.screeny = touch.clientY;
-            return;
-        }
-        else if (evt.touches.length === 2) {
-            this._didMultitouch = true;
-            const touch1 = evt.touches[0];
-            const touch2 = evt.touches[1];
-            const prev1 = this.prev.get(touch1.identifier);
-            const prev2 = this.prev.get(touch2.identifier);
-            if (!prev1 || !prev2) return;
-
-            if (this.twoFingerRotate) {
-                const angle1 = Math.atan2(touch1.clientY - touch2.clientY, touch1.clientX - touch2.clientX);
-                const lastAngle = Math.atan2(prev1.screeny - prev2.screeny, prev1.screenx - prev2.screenx);
-                const angleDiff = angle1 - lastAngle;
-                if (Math.abs(angleDiff) > 0.001) {
-                    this.addRotation(angleDiff);
-                }
-            }
-
-            if (this.twoFingerScale) {
-                const distx = touch1.clientX - touch2.clientX;
-                const disty = touch1.clientY - touch2.clientY;
-                const dist = Math.sqrt(distx * distx + disty * disty);
-                const lastDistx = prev1.screenx - prev2.screenx;
-                const lastDisty = prev1.screeny - prev2.screeny;
-                const lastDist = Math.sqrt(lastDistx * lastDistx + lastDisty * lastDisty);
-                const distDiff = dist - lastDist;
-                if (Math.abs(distDiff) > 2) {
-                    this.addScale(distDiff)
-                }
-            }
-
-            prev1.screenx = touch1.clientX;
-            prev1.screeny = touch1.clientY;
-            prev2.screenx = touch2.clientX;
-            prev2.screeny = touch2.clientY;
-        }
-    }
-
-    private readonly _raycaster: Raycaster = new Raycaster();
-    private readonly _intersection: Vector3 = new Vector3();
-    private readonly _screenPos: Vector3 = new Vector3();
-
-    private getPositionOnPlane(tx: number, ty: number): Vector3 {
-        const camera = this.context.mainCamera!;
-        this._screenPos.x = (tx / window.innerWidth) * 2 - 1;
-        this._screenPos.y = -(ty / window.innerHeight) * 2 + 1;
-        this._screenPos.z = 1;
-
-        this._screenPos.unproject(camera);
-        this._raycaster.set(camera.position, this._screenPos.sub(camera.position));
-        this._raycaster.ray.intersectPlane(this.plane, this._intersection);
-        return this._intersection;
-    }
-
-    private addMovement(dx: number, dz: number) {
-        // this.translate.x -= dx;
-        // this.translate.z -= dz;
-        // this._needsUpdate = true;
-        // return
-        // some arbitrary factor
-        dx *= .75;
-        dz *= .75;
-        // increase diff if the scene is scaled small
-        dx /= this._scale;
-        dz /= this._scale;
-        // apply it
-        this.offset.elements[12] -= dx;
-        this.offset.elements[14] -= dz;
-    };
-
-    private readonly _tempMatrix: Matrix4 = new Matrix4();
-
-    private addScale(diff: number) {
-        diff /= window.innerWidth
-
-        // this.scale.x *= 1 + diff;
-        // this.scale.y *= 1 + diff;
-        // this.scale.z *= 1 + diff;
-        // this._needsUpdate = true;
-        // return;
-
-
-        // we use this factor to modify the translation factor (in apply movement)
-        this._scale *= 1 + diff;
-        // apply the scale
-        this._tempMatrix.makeScale(1 - diff, 1 - diff, 1 - diff);
-        this.offset.premultiply(this._tempMatrix);
-    }
-
-
-    private addRotation(rot: number) {
-        // this.rotation.multiply(new Quaternion().setFromAxisAngle(WebXRSessionRootUserInput.up, rot));
-        // this._needsUpdate = true;
-        // return;
-        this._tempMatrix.makeRotationY(rot);
-        this.offset.premultiply(this._tempMatrix);
-    }
-}
+import { Behaviour, GameObject } from "../Component.js";
+import { Matrix4, Object3D, Plane, Quaternion, Ray, Raycaster, Vector2, Vector3 } from "three";
+import { WebAR, WebXR } from "./WebXR.js";
+import { InstancingUtil } from "../../engine/engine_instancing.js";
+import { serializable } from "../../engine/engine_serialization_decorator.js";
+import { Context } from "../../engine/engine_context.js";
+import { isQuest } from "../../engine/engine_utils.js";
+
+// https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
+
+const tempMatrix = new Matrix4();
+
+export class WebARSessionRoot extends Behaviour {
+
+    webAR: WebAR | null = null;
+
+    get rig(): Object3D | undefined {
+        return this.webAR?.webxr.Rig;
+    }
+
+    @serializable()
+    invertForward: boolean = false;
+
+    /** Preview feature: enable touch transform */
+    @serializable()
+    arTouchTransform: boolean = false;
+
+    @serializable()
+    get arScale(): number {
+        return this._arScale;
+    }
+    set arScale(val: number) {
+        if (val === this._arScale) return;
+        this._arScale = val;
+        this.setScale(val);
+    }
+
+    private readonly _initalMatrix = new Matrix4();
+    private readonly _selectStartFn = this.onSelectStart.bind(this);
+    private readonly _selectEndFn = this.onSelectEnd.bind(this);
+    private userInput?: WebXRSessionRootUserInput;
+
+    start() {
+        const xr = GameObject.findObjectOfType(WebXR);
+        if (xr) {
+            xr.Rig.updateMatrix();
+            this._initalMatrix.copy(xr.Rig.matrix);
+        }
+    }
+
+    private _arScale: number = 1;
+    private _rig: Object3D | null = null;
+    private _startPose: Matrix4 | null = null;
+    private _placementPose: Matrix4 | null = null;
+    private _isTouching: boolean = false;
+    private _rigStartPose: Matrix4 | undefined | null = null;
+    private _gotFirstHitTestResult: boolean = false;
+    private _anchor: XRAnchor | null = null;
+
+    onBegin(session: XRSession) {
+
+        this._placementPose = null;
+        this.gameObject.visible = false;
+        this.gameObject.matrixAutoUpdate = false;
+        this._startPose = this.gameObject.matrix.clone();
+        this._rigStartPose = this.rig?.matrix.clone();
+        this._gotFirstHitTestResult = false;
+        this._anchor = null;
+        session.addEventListener('selectstart', this._selectStartFn);
+        session.addEventListener('selectend', this._selectEndFn);
+        // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
+
+        // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
+        this.gameObject.visible = false;
+
+        if (this.rig) {
+            // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
+            // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
+            // immersive Ar right now to place the user/offset the session
+            this.rig.matrixAutoUpdate = true;
+            this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
+        }
+
+        // TODO this is duplicate to WebXR events AND engine events, would be better in one place
+        this.dispatchEvent(new CustomEvent('onBeginSession'));
+    }
+
+    onUpdate(rig: Object3D | null, _session: XRSession, hit: XRHitTestResult | null, pose: XRPose | null | undefined): boolean {
+
+        if (pose && !this._placementPose) {
+            if (!this._gotFirstHitTestResult) {
+                this._gotFirstHitTestResult = true;
+                this.dispatchEvent(new CustomEvent('canPlaceSession', { detail: { pose: pose } }));
+            }
+
+            if (this._isTouching) {
+                // callbacks
+                const poseMatrix = new Matrix4().fromArray(pose.transform.matrix).invert();
+                this.dispatchEvent(new CustomEvent('placedSession', { detail: { pose, poseMatrix } }));
+
+                if (this.webAR) this.webAR.setReticleActive(false);
+                this.placeAt(rig, poseMatrix);
+                if (hit && pose && !isQuest()) // TODO anchors seem to behave differently with an XRRig
+                   this.onCreatePlacementAnchor(hit, pose);
+
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private onCreatePlacementAnchor(hit: XRHitTestResult, pose: XRPose) {
+        this._anchor = null;
+        hit?.createAnchor?.call(hit, pose.transform)?.then(anchor => {
+            if (this.context.isInAR)
+                this._anchor = anchor;
+        }).catch(ex => {
+            console.warn("Failed to create anchor", ex);
+        });
+    }
+
+    private _anchorMatrix: Matrix4 = new Matrix4();
+
+    onBeforeRender(frame: XRFrame | null): void {
+        if (frame && this._rig) {
+            if (this._anchor) {
+                const referenceSpace = this.context.renderer.xr.getReferenceSpace();
+                if (referenceSpace) {
+                    const pose = frame.getPose(this._anchor.anchorSpace, referenceSpace);
+                    if (pose) {
+                        const poseMatrix = this._anchorMatrix.fromArray(pose.transform.matrix).invert();
+                        this.placeAt(this._rig, poseMatrix);
+                        return;
+                    }
+                }
+            }
+            else if (this._placementPose) {
+                this.placeAt(this._rig, this._placementPose!);
+            }
+        }
+    }
+
+    private _invertedSessionRootMatrix: Matrix4 = new Matrix4();
+    private _invertForwardMatrix: Matrix4 = new Matrix4().makeRotationY(Math.PI);
+
+    placeAt(rig: Object3D | null, mat: Matrix4) {
+        if (!this._placementPose) this._placementPose = new Matrix4();
+        this._placementPose.copy(mat);
+
+        // apply session root offset
+        const invertedSessionRoot = this._invertedSessionRootMatrix.copy(this.gameObject.matrixWorld).invert();
+        this._placementPose.premultiply(invertedSessionRoot);
+        if (rig) {
+            if (this.invertForward) {
+                this._placementPose.premultiply(this._invertForwardMatrix);
+            }
+
+            if (!this.userInput) this.userInput = new WebXRSessionRootUserInput(this.context);
+            this.userInput.enable();
+
+            this._rig = rig;
+            this.setScale(this.arScale);
+        }
+        else this._rig = null;
+        this.gameObject.visible = true;
+    }
+
+    onEnd(rig: Object3D | null, _session: XRSession) {
+        this.userInput?.disable();
+        this.userInput?.reset();
+
+        this._placementPose = null;
+        this.gameObject.visible = false;
+        this.gameObject.matrixAutoUpdate = false;
+        this._anchor = null;
+        if (this._startPose) {
+            this.gameObject.matrix.copy(this._startPose);
+        }
+        if (rig) {
+            rig.matrixAutoUpdate = true;
+            if (this._rigStartPose) {
+                this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
+                // console.log(rig.position, rig.quaternion, rig.scale);
+            }
+        }
+        InstancingUtil.markDirty(this.gameObject, true);
+        // HACK to fix physics being not in correct place after exiting AR
+        setTimeout(() => {
+            if (!this.gameObject) return;
+            this.gameObject.matrixAutoUpdate = true;
+            this.gameObject.visible = true;
+        }, 100);
+    }
+
+
+    private onSelectStart() {
+        this._isTouching = true;
+    }
+
+    private onSelectEnd() {
+        this._isTouching = false;
+    }
+
+    private setScale(scale) {
+        const rig = this._rig;
+        if (!rig || !this._placementPose) {
+            return;
+        }
+        // Capture the rig position before the first time we move it during a session
+        if (!this._rigStartPose) {
+            this._rigStartPose = rig.matrix.clone();
+        }
+        // we apply the transform to the rig because we want to move the user's position for easy networking
+        rig.matrixAutoUpdate = false;
+        if (this.arTouchTransform && this.userInput) {
+            this.userInput.applyMatrixTo(this._placementPose);
+            // rig.matrix.premultiply(this.userInput.offset);
+        }
+        rig.matrix.multiplyMatrices(tempMatrix.makeScale(scale, scale, scale), this._placementPose);
+        rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
+        rig.updateMatrixWorld();
+    }
+}
+
+
+
+
+class WebXRSessionRootUserInput {
+    private static up = new Vector3(0, 1, 0);
+    private static zero = new Vector3(0, 0, 0);
+    private static one = new Vector3(1, 1, 1);
+
+    oneFingerDrag: boolean = true;
+    twoFingerRotate: boolean = true;
+    twoFingerScale: boolean = true;
+
+    readonly context: Context;
+    readonly offset: Matrix4;
+    readonly plane: Plane;
+
+    private _scale: number = 1;
+
+    // readonly translate: Vector3 = new Vector3();
+    // readonly rotation: Quaternion = new Quaternion();
+    // readonly scale: Vector3 = new Vector3(1, 1, 1);
+
+    constructor(context: Context) {
+        this.context = context;
+        this.offset = new Matrix4()
+        this.plane = new Plane();
+        this.plane.setFromNormalAndCoplanarPoint(WebXRSessionRootUserInput.up, WebXRSessionRootUserInput.zero);
+    }
+
+    private _enabled: boolean = false;
+    enable() {
+        if (this._enabled) return;
+        this._enabled = true;
+        window.addEventListener('touchstart', this.touchStart, { passive: false });
+        window.addEventListener('touchmove', this.touchMove, { passive: false });
+        window.addEventListener('touchend', this.touchEnd, { passive: false });
+    }
+    disable() {
+        if (!this._enabled) return;
+        this._enabled = false;
+        window.removeEventListener('touchstart', this.touchStart);
+        window.removeEventListener('touchmove', this.touchMove);
+        window.removeEventListener('touchend', this.touchEnd);
+    }
+    reset() {
+        this._scale = 1;
+        this.offset.identity();
+    }
+    applyMatrixTo(matrix: Matrix4) {
+        matrix.premultiply(this.offset);
+        // if (this._needsUpdate)
+        //     this.updateMatrix();
+        // matrix.premultiply(this._rotationMatrix);
+        // matrix.premultiply(this.offset).premultiply(this._rotationMatrix)
+    }
+
+    // private _needsUpdate: boolean = true;
+    // private _rotationMatrix: Matrix4 = new Matrix4();
+    // private updateMatrix() {
+    //     this._needsUpdate = false;
+    //     this._rotationMatrix.makeRotationFromQuaternion(this.rotation);
+    //     this.offset.compose(this.translate, new Quaternion(), this.scale);
+    //     // const rot = this._tempMatrix.makeRotationY(this.angle);
+    //     // this.translate.applyMatrix4(rot);
+    //     // this.offset.elements[12] = this.translate.x;
+    //     // this.offset.elements[13] = this.translate.y;
+    //     // this.offset.elements[14] = this.translate.z;
+    //     // this.offset.premultiply(rot);
+    //     // const s = this.scale;
+    //     // this.offset.premultiply(this._tempMatrix.makeScale(s, s, s));
+    // }
+
+    private prev: Map<number, { x: number, z: number, screenx: number, screeny: number }> = new Map();
+    private _didMultitouch: boolean = false;
+
+    private touchStart = (evt: TouchEvent) => {
+        for (let i = 0; i < evt.changedTouches.length; i++) {
+            const touch = evt.changedTouches[i];
+            if (!this.prev.has(touch.identifier))
+                this.prev.set(touch.identifier, {
+                    x: 0,
+                    z: 0,
+                    screenx: 0,
+                    screeny: 0,
+                });
+            const prev = this.prev.get(touch.identifier);
+            if (prev) {
+                const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
+                prev.x = pos.x;
+                prev.z = pos.z;
+                prev.screenx = touch.clientX;
+                prev.screeny = touch.clientY;
+            }
+        }
+    }
+    private touchEnd = (evt: TouchEvent) => {
+        if (evt.touches.length <= 0) {
+            this._didMultitouch = false;
+        }
+    }
+    private touchMove = (evt: TouchEvent) => {
+        if (evt.defaultPrevented) return;
+        
+        if (evt.touches.length === 1) {
+            // if we had multiple touches before due to e.g. pinching / rotating
+            // and stopping one of the touches, we don't want to move the scene suddenly
+            // this will be resettet when all touches stop
+            if (this._didMultitouch) {
+                return;
+            }
+            const touch = evt.touches[0];
+            const prev = this.prev.get(touch.identifier);
+            if (!prev) return;
+            const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
+            const dx = pos.x - prev.x;
+            const dy = pos.z - prev.z;
+            if (dx === 0 && dy === 0) return;
+            if (this.oneFingerDrag)
+                this.addMovement(dx, dy);
+            prev.x = pos.x;
+            prev.z = pos.z;
+            prev.screenx = touch.clientX;
+            prev.screeny = touch.clientY;
+            return;
+        }
+        else if (evt.touches.length === 2) {
+            this._didMultitouch = true;
+            const touch1 = evt.touches[0];
+            const touch2 = evt.touches[1];
+            const prev1 = this.prev.get(touch1.identifier);
+            const prev2 = this.prev.get(touch2.identifier);
+            if (!prev1 || !prev2) return;
+
+            if (this.twoFingerRotate) {
+                const angle1 = Math.atan2(touch1.clientY - touch2.clientY, touch1.clientX - touch2.clientX);
+                const lastAngle = Math.atan2(prev1.screeny - prev2.screeny, prev1.screenx - prev2.screenx);
+                const angleDiff = angle1 - lastAngle;
+                if (Math.abs(angleDiff) > 0.001) {
+                    this.addRotation(angleDiff);
+                }
+            }
+
+            if (this.twoFingerScale) {
+                const distx = touch1.clientX - touch2.clientX;
+                const disty = touch1.clientY - touch2.clientY;
+                const dist = Math.sqrt(distx * distx + disty * disty);
+                const lastDistx = prev1.screenx - prev2.screenx;
+                const lastDisty = prev1.screeny - prev2.screeny;
+                const lastDist = Math.sqrt(lastDistx * lastDistx + lastDisty * lastDisty);
+                const distDiff = dist - lastDist;
+                if (Math.abs(distDiff) > 2) {
+                    this.addScale(distDiff)
+                }
+            }
+
+            prev1.screenx = touch1.clientX;
+            prev1.screeny = touch1.clientY;
+            prev2.screenx = touch2.clientX;
+            prev2.screeny = touch2.clientY;
+        }
+    }
+
+    private readonly _raycaster: Raycaster = new Raycaster();
+    private readonly _intersection: Vector3 = new Vector3();
+    private readonly _screenPos: Vector3 = new Vector3();
+
+    private getPositionOnPlane(tx: number, ty: number): Vector3 {
+        const camera = this.context.mainCamera!;
+        this._screenPos.x = (tx / window.innerWidth) * 2 - 1;
+        this._screenPos.y = -(ty / window.innerHeight) * 2 + 1;
+        this._screenPos.z = 1;
+
+        this._screenPos.unproject(camera);
+        this._raycaster.set(camera.position, this._screenPos.sub(camera.position));
+        this._raycaster.ray.intersectPlane(this.plane, this._intersection);
+        return this._intersection;
+    }
+
+    private addMovement(dx: number, dz: number) {
+        // this.translate.x -= dx;
+        // this.translate.z -= dz;
+        // this._needsUpdate = true;
+        // return
+        // some arbitrary factor
+        dx *= .75;
+        dz *= .75;
+        // increase diff if the scene is scaled small
+        dx /= this._scale;
+        dz /= this._scale;
+        // apply it
+        this.offset.elements[12] -= dx;
+        this.offset.elements[14] -= dz;
+    };
+
+    private readonly _tempMatrix: Matrix4 = new Matrix4();
+
+    private addScale(diff: number) {
+        diff /= window.innerWidth
+
+        // this.scale.x *= 1 + diff;
+        // this.scale.y *= 1 + diff;
+        // this.scale.z *= 1 + diff;
+        // this._needsUpdate = true;
+        // return;
+
+
+        // we use this factor to modify the translation factor (in apply movement)
+        this._scale *= 1 + diff;
+        // apply the scale
+        this._tempMatrix.makeScale(1 - diff, 1 - diff, 1 - diff);
+        this.offset.premultiply(this._tempMatrix);
+    }
+
+
+    private addRotation(rot: number) {
+        // this.rotation.multiply(new Quaternion().setFromAxisAngle(WebXRSessionRootUserInput.up, rot));
+        // this._needsUpdate = true;
+        // return;
+        this._tempMatrix.makeRotationY(rot);
+        this.offset.premultiply(this._tempMatrix);
+    }
+}