@needle-tools/engine 4.7.0-alpha → 4.7.0-next.2756168

package/src/engine-components/webxr/WebARSessionRoot.ts

@@ -1,880 +1,880 @@
-import { AxesHelper, DoubleSide, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, Raycaster, RingGeometry, Scene, Vector3 } from "three";
-
-import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
-import { AssetReference } from "../../engine/engine_addressables.js";
-import { Context } from "../../engine/engine_context.js";
-import { destroy, instantiate } from "../../engine/engine_gameobject.js";
-import { InputEventQueue, NEPointerEvent } from "../../engine/engine_input.js";
-import { getBoundingBox, getTempVector } from "../../engine/engine_three_utils.js";
-import type { IComponent, IGameObject } from "../../engine/engine_types.js";
-import { DeviceUtilities, getParam } from "../../engine/engine_utils.js";
-import { NeedleXRController, type NeedleXREventArgs, type NeedleXRHitTestResult, NeedleXRSession } from "../../engine/engine_xr.js";
-import { Behaviour, GameObject } from "../Component.js";
-
-// https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
-
-const debug = getParam("debugwebxr");
-
-const invertForwardMatrix = new Matrix4().makeRotationY(Math.PI);
-
-
-/**
- * The WebARSessionRoot is the root object for a WebAR session and used to place the scene in AR.  
- * It is also responsible for scaling the user in AR and optionally creating a XR anchor for the scene placement.  
- * @example 
- * ```ts
- * WebARSessionRoot.onPlaced((args) => {
- *    console.log("Scene has been placed in AR");
- * });
- * ```
- * 
- * @category XR
- * @group Components
- */
-export class WebARSessionRoot extends Behaviour {
-
-    private static _eventListeners: { [key: string]: Array<(args: { instance: WebARSessionRoot }) => void> } = {};
-    /**
-     * Event that is called when the scene has been placed in AR.
-     * @param cb the callback that is called when the scene has been placed
-     * @returns a function to remove the event listener
-     */
-    static onPlaced(cb: (args: { instance: WebARSessionRoot }) => void) {
-        const event = "placed";
-        if (!this._eventListeners[event]) this._eventListeners[event] = [];
-        this._eventListeners[event].push(cb);
-        return () => {
-            const index = this._eventListeners[event].indexOf(cb);
-            if (index >= 0) this._eventListeners[event].splice(index, 1);
-        }
-    }
-
-    private static _hasPlaced: boolean = false;
-    /**
-     * @returns true if the scene has been placed in AR by the user or automatic placement
-     */
-    static get hasPlaced(): boolean {
-        return this._hasPlaced;
-    }
-
-
-    /** The scale of the user in AR.  
-     * **NOTE**: a large value makes the scene appear smaller  
-     * @default 1
-     */
-    get arScale(): number {
-        return this._arScale;
-    }
-    set arScale(val: number) {
-        this._arScale = Math.max(0.000001, val);
-        this.onSetScale();
-    }
-    private _arScale: number = 1;
-
-    /** When enabled the placed scene forward direction will towards the XRRig 
-     * @deprecated
-     * @default false
-    */
-    invertForward: boolean = false;
-
-    /** When assigned this asset will be loaded and visualize the placement while in AR
-     * @default null
-     */
-    customReticle?: AssetReference;
-
-    /** Enable touch transform to translate, rotate and scale the scene in AR with multitouch
-     * @default true
-     */
-    arTouchTransform: boolean = true;
-
-    /** When enabled the scene will be placed automatically when a point in the real world is found
-     * @default false
-     */
-    autoPlace: boolean = false;
-
-    /** When enabled the scene center will be automatically calculated from the content in the scene */
-    autoCenter: boolean = false;
-
-    /** Experimental: When enabled we will create a XR anchor for the scene placement    
-     * and make sure the scene is at that anchored point during a XR session 
-     * @default false
-     **/
-    useXRAnchor: boolean = false;
-
-    /** true if we're currently placing the scene */
-    private _isPlacing = true;
-
-    /** This is the world matrix of the ar session root when entering webxr
-     * it is applied when the scene has been placed (e.g. if the session root is x:10, z:10 we want this position to be the center of the scene)
-     */
-    private readonly _startOffset: Matrix4 = new Matrix4();
-
-    private _createdPlacementObject: Object3D | null = null;
-    private readonly _reparentedComponents: Array<{ comp: IComponent, originalObject: IGameObject }> = [];
-
-    // move objects into a temporary scene while placing (which is not rendered) so that the components won't be disabled during this process
-    // e.g. we want the avatar to still be updated while placing
-    // another possibly solution would be to ensure from this component that the Rig is *also* not disabled while placing
-    private readonly _placementScene: Scene = new Scene();
-
-    /** the reticles used for placement */
-    private readonly _reticle: IGameObject[] = [];
-    /** needs to be in sync with the reticles */
-    private readonly _hits: XRHitTestResult[] = [];
-
-    private _placementStartTime: number = -1;
-    private _rigPlacementMatrix?: Matrix4;
-    /** if useAnchor is enabled this is the anchor we have created on placing the scene using the placement hit */
-    private _anchor: XRAnchor | null = null;
-    /** user input is used for ar touch transform */
-    private userInput?: WebXRSessionRootUserInput;
-
-    onEnable(): void {
-        this.customReticle?.preload();
-    }
-
-    supportsXR(mode: XRSessionMode): boolean {
-        return mode === "immersive-ar";
-    }
-
-    onEnterXR(_args: NeedleXREventArgs): void {
-        if (debug) console.log("ENTER WEBXR: SessionRoot start...");
-
-        this._anchor = null;
-        WebARSessionRoot._hasPlaced = false;
-
-        // if (_args.xr.session.enabledFeatures?.includes("image-tracking")) {
-        //     console.warn("Image tracking is enabled - will not place scene");
-        //     return;
-        // }
-
-        // save the transform of the session root in the scene to apply it when placing the scene
-        this.gameObject.updateMatrixWorld();
-        this._startOffset.copy(this.gameObject.matrixWorld);
-
-        // create a new root object for the session placement scripts
-        // and move all the children in the scene in a temporary scene that is not rendered
-        const rootObject = new Object3D();
-        this._createdPlacementObject = rootObject;
-        rootObject.name = "AR Session Root";
-        this._placementScene.name = "AR Placement Scene";
-        this._placementScene.children.length = 0;
-        for (let i = this.context.scene.children.length - 1; i >= 0; i--) {
-            const ch = this.context.scene.children[i];
-            this._placementScene.add(ch);
-        }
-        this.context.scene.add(rootObject);
-
-        if (this.autoCenter) {
-            const bounds = getBoundingBox(this._placementScene.children);
-            const center = bounds.getCenter(new Vector3());
-            const size = bounds.getSize(new Vector3());
-            const matrix = new Matrix4();
-            matrix.makeTranslation(center.x, center.y - size.y * .5, center.z);
-            this._startOffset.multiply(matrix);
-        }
-
-        // reparent components
-        // save which gameobject the sessionroot component was previously attached to
-        this._reparentedComponents.length = 0;
-        this._reparentedComponents.push({ comp: this, originalObject: this.gameObject });
-        GameObject.addComponent(rootObject, this);
-        // const webXR = GameObject.findObjectOfType(WebXR2);
-        // if (webXR) {
-        //     this._reparentedComponents.push({ comp: webXR, originalObject: webXR.gameObject });
-        //     GameObject.addComponent(rootObject, webXR);
-        //     const playerSync = GameObject.findObjectOfType(XRFlag);
-        // }
-
-        // recreate the reticle every time we enter AR
-        for (const ret of this._reticle) {
-            destroy(ret);
-        }
-        this._reticle.length = 0;
-        this._isPlacing = true;
-        // we want to receive pointer events EARLY and prevent interaction with other objects while placing by stopping the event propagation
-        this.context.input.addEventListener("pointerup", this.onPlaceScene, { queue: InputEventQueue.Early });
-    }
-    onLeaveXR() {
-        // TODO: WebARSessionRoot doesnt work when we enter passthrough and leave XR without having placed the session!!!
-        this.context.input.removeEventListener("pointerup", this.onPlaceScene, { queue: InputEventQueue.Early });
-        this.onRevertSceneChanges();
-        // this._anchor?.delete();
-        this._anchor = null;
-        WebARSessionRoot._hasPlaced = false;
-        this._rigPlacementMatrix = undefined;
-    }
-    onUpdateXR(args: NeedleXREventArgs): void {
-
-        // disable session placement while images are being tracked
-        if (args.xr.isTrackingImages) {
-            for (const ret of this._reticle)
-                ret.visible = false;
-            return;
-        }
-
-        if (this._isPlacing) {
-            const rigObject = args.xr.rig?.gameObject;
-            // the rig should be parented to the scene while placing
-            // since the camera is always parented to the rig this ensures that the camera is always rendering
-            if (rigObject && rigObject.parent !== this.context.scene) {
-                this.context.scene.add(rigObject);
-            }
-
-            // in pass through mode we want to place the scene using an XR controller
-            let controllersDidHit = false;
-            // when auto placing we just use the user's view
-            if (args.xr.isPassThrough && args.xr.controllers.length > 0 && !this.autoPlace) {
-                for (const ctrl of args.xr.controllers) {
-                    // with this we can only place with the left / first controller right now
-                    // we also only have one reticle... this should probably be refactored a bit so we can have multiple reticles
-                    // and then place at the reticle for which the user clicked the place button
-                    const hit = ctrl.getHitTest();
-                    if (hit) {
-                        controllersDidHit = true;
-                        this.updateReticleAndHits(args.xr, ctrl.index, hit, args.xr.rigScale);
-                    }
-                }
-            }
-            // in screen AR mode we use "camera" hit testing (or when using the simulator where controller hit testing is not supported)
-            if (!controllersDidHit) {
-                const hit = args.xr.getHitTest();
-                if (hit) {
-                    this.updateReticleAndHits(args.xr, 0, hit, args.xr.rigScale);
-                }
-            }
-
-        }
-        else {
-            // Update anchors, if any
-            if (this._anchor && args.xr.referenceSpace) {
-                const pose = args.xr.frame.getPose(this._anchor.anchorSpace, args.xr.referenceSpace);
-                if (pose && this.context.time.frame % 20 === 0) {
-                    // apply the anchor pose to one of the reticles
-                    const converted = args.xr.convertSpace(pose.transform);
-                    const reticle = this._reticle[0];
-                    if (reticle) {
-                        reticle.position.copy(converted.position);
-                        reticle.quaternion.copy(converted.quaternion);
-                        this.onApplyPose(reticle);
-                    }
-                }
-            }
-
-            // Scene has been placed
-            if (this.arTouchTransform) {
-                if (!this.userInput) this.userInput = new WebXRSessionRootUserInput(this.context);
-                this.userInput?.enable();
-            }
-            else this.userInput?.disable();
-            if (this.arTouchTransform && this.userInput?.hasChanged) {
-                if (args.xr.rig) {
-                    const rig = args.xr.rig.gameObject;
-                    this.userInput.applyMatrixTo(rig.matrix, true);
-                    rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
-                    // if the rig is scaled large we want the drag touch to be faster
-                    this.userInput.factor = rig.scale.x;
-                }
-                this.userInput.reset();
-            }
-        }
-    }
-
-    private updateReticleAndHits(_xr: NeedleXRSession, i: number, hit: NeedleXRHitTestResult, scale: number) {
-        // save the hit test
-        this._hits[i] = hit.hit;
-        let reticle = this._reticle[i];
-        if (!reticle) {
-            if (this.customReticle) {
-                if (this.customReticle.asset) {
-                    reticle = instantiate(this.customReticle.asset);
-                }
-                else {
-                    this.customReticle.loadAssetAsync();
-                    return;
-                }
-            }
-            else {
-                reticle = new Mesh(
-                    new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
-                    new MeshBasicMaterial({ side: DoubleSide, depthTest: false, depthWrite: false, transparent: true, opacity: 1, color: 0xeeeeee })
-                ) as any as IGameObject;
-                reticle.name = "AR Placement Reticle";
-            }
-            if (debug) {
-                const axes = new AxesHelper(1);
-                axes.position.y += .01;
-                reticle.add(axes);
-            }
-            this._reticle[i] = reticle;
-            reticle.matrixAutoUpdate = false;
-            reticle.visible = false;
-        }
-        reticle["lastPos"] = reticle["lastPos"] || hit.position.clone();
-        reticle["lastQuat"] = reticle["lastQuat"] || hit.quaternion.clone();
-        // reticle["targetPos"] = reticle["targetPos"] || hit.position.clone();
-        // reticle["targetQuat"] = reticle["targetQuat"] || hit.quaternion.clone();
-
-        // TODO we likely want the reticle itself to be placed _exactly_ and then the visuals being lerped,
-        // Right now this leads to a "rotation glitch" when going from a horizontal to a vertical surface
-        reticle.position.copy(reticle["lastPos"].lerp(hit.position, this.context.time.deltaTime / .1));
-        reticle["lastPos"].copy(reticle.position);
-        reticle.quaternion.copy(reticle["lastQuat"].slerp(hit.quaternion, this.context.time.deltaTime / .05));
-        reticle["lastQuat"].copy(reticle.quaternion);
-
-        // TODO make sure original reticle asset scale is respected, or document it should be uniformly scaled
-        // scale *= this.customReticle?.asset?.scale?.x || 1;
-        reticle.scale.set(scale, scale, scale);
-
-        // if (this.invertForward) {
-        //     reticle.rotateY(Math.PI);
-        // }
-
-        // Workaround: For a custom reticle we apply the view based transform during placement preview
-        // See NE-4161 for context
-        if (this.customReticle)
-            this.applyViewBasedTransform(reticle);
-
-        reticle.updateMatrix();
-        reticle.visible = true;
-        if (reticle.parent !== this.context.scene)
-            this.context.scene.add(reticle);
-
-        if (this._placementStartTime < 0) {
-            this._placementStartTime = this.context.time.realtimeSinceStartup;
-        }
-
-        if (this.autoPlace) {
-            this.upVec.set(0, 1, 0).applyQuaternion(reticle.quaternion);
-            const isUp = this.upVec.dot(getTempVector(0, 1, 0)) > 0.9;
-            if (isUp) {
-                // We want the reticle to be at a suitable spot for a moment before we place the scene (not place it immediately)
-                let autoplace_timer = reticle["autoplace:timer"] || 0;
-                if (autoplace_timer >= 1) {
-                    reticle.visible = false;
-                    this.onPlaceScene(null);
-                }
-                else {
-                    autoplace_timer += this.context.time.deltaTime;
-                    reticle["autoplace:timer"] = autoplace_timer;
-                }
-            }
-            else {
-                reticle["autoplace:timer"] = 0;
-            }
-        }
-    }
-
-    private onPlaceScene = (evt: NEPointerEvent | null) => {
-        if (this._isPlacing == false) return;
-        if (evt?.used) return;
-
-        let reticle: IGameObject | undefined = this._reticle[0];
-
-        if (!reticle) {
-            console.warn("No reticle to place...");
-            return;
-        }
-
-        if (!reticle.visible && !this.autoPlace) {
-            console.warn("Reticle is not visible (can not place)");
-            return;
-        }
-
-        if (NeedleXRSession.active?.isTrackingImages) {
-            console.warn("Scene Placement is disabled while images are being tracked");
-            return;
-        }
-
-        let hit = this._hits[0];
-
-        if (evt && evt.origin instanceof NeedleXRController) {
-            // until we can use hit testing for both controllers and have multple reticles we only allow placement with the first controller
-            const controllerReticle = this._reticle[evt.origin.index];
-            if (controllerReticle) {
-                reticle = controllerReticle;
-                hit = this._hits[evt.origin.index];
-            }
-        }
-
-        // if we place the scene we don't want this event to be propagated to any sub-objects (via the EventSystem) anymore and trigger e.g. a click on objects for the "place tap" event
-        if (evt) {
-            evt.stopImmediatePropagation();
-            evt.stopPropagation();
-            evt.use();
-        }
-
-        this._isPlacing = false;
-        this.context.input.removeEventListener("pointerup", this.onPlaceScene);
-
-        this.onRevertSceneChanges();
-
-        // TODO: we should probably use the non-lerped position and quaternion here
-        reticle.position.copy(reticle["lastPos"]);
-        reticle.quaternion.copy(reticle["lastQuat"]);
-
-        this.onApplyPose(reticle);
-        WebARSessionRoot._hasPlaced = true;
-
-        if (this.useXRAnchor) {
-            this.onCreateAnchor(NeedleXRSession.active!, hit);
-        }
-
-        if (this.context.xr) {
-            for (const ctrl of this.context.xr.controllers) {
-                ctrl.cancelHitTestSource();
-            }
-        }
-    }
-
-    private onSetScale() {
-        if (!WebARSessionRoot._hasPlaced) return;
-        const rig = NeedleXRSession.active?.rig?.gameObject;
-        if (rig) {
-            const currentScale = NeedleXRSession.active?.rigScale || 1;
-            const newScale = (1 / this._arScale) * currentScale;
-            const scaleMatrix = new Matrix4().makeScale(newScale, newScale, newScale).invert();
-            rig.matrix.premultiply(scaleMatrix);
-            rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
-        }
-    }
-
-    private onRevertSceneChanges() {
-        for (const ret of this._reticle) {
-            if (!ret) continue;
-            ret.visible = false;
-            ret?.removeFromParent();
-        }
-        this._reticle.length = 0;
-
-        for (let i = this._placementScene.children.length - 1; i >= 0; i--) {
-            const ch = this._placementScene.children[i];
-            this.context.scene.add(ch);
-        }
-        this._createdPlacementObject?.removeFromParent();
-
-        for (const reparented of this._reparentedComponents) {
-            GameObject.addComponent(reparented.originalObject, reparented.comp);
-        }
-    }
-
-    private async onCreateAnchor(session: NeedleXRSession, hit: XRHitTestResult) {
-        if (hit.createAnchor === undefined) {
-            console.warn("Hit does not support creating an anchor", hit);
-            if (isDevEnvironment()) showBalloonWarning("Hit does not support creating an anchor");
-            return;
-        }
-        else {
-            const anchor = await hit.createAnchor(session.viewerPose!.transform);
-            // make sure the session is still active
-            if (session.running && anchor) {
-                this._anchor = anchor;
-            }
-        }
-    }
-
-    private upVec: Vector3 = new Vector3(0, 1, 0);
-    private lookPoint: Vector3 = new Vector3();
-    private worldUpVec: Vector3 = new Vector3(0, 1, 0);
-
-    private applyViewBasedTransform(reticle: Object3D) {
-
-        // Make reticle face the user to unify the placement experience across devices.
-        // The pose that we're receiving from the hit test varies between devices:
-        // - Quest: currently aligned to the mesh that was hit (depends on room setup), has changed a couple times
-        // - Android WebXR: looking at the camera, but pretty random when on a wall
-        // - Mozilla WebXR Viewer: aligned to the start of the session
-        const camGo = this.context.mainCamera as Object3D as GameObject;
-        const reticleGo = reticle as GameObject;
-        const camWP = camGo.worldPosition;
-        const reticleWp = reticleGo.worldPosition;
-
-        this.upVec.set(0, 1, 0).applyQuaternion(reticle.quaternion);
-
-        // upVec may be pointing AWAY from us, we have to flip it if that's the case
-        const camPos = camGo.worldPosition;
-        if (camPos) {
-            const camToReticle = reticle.position.clone().sub(camPos);
-            const angle = camToReticle.angleTo(this.upVec);
-            if (angle < Math.PI / 2) {
-                this.upVec.negate();
-            }
-        }
-
-        const upAngle = this.upVec.angleTo(this.worldUpVec) * 180 / Math.PI;
-        // For debugging look angle for AR placement
-        // Gizmos.DrawDirection(reticle.position, upVec, "blue", 0.1);
-        // Gizmos.DrawLabel(reticle.position, upAngle.toFixed(2), 0.1);
-
-        const angleForWallPlacement = 30;
-        if ((upAngle > angleForWallPlacement && upAngle < 180 - angleForWallPlacement) ||
-            (upAngle < -angleForWallPlacement && upAngle > -180 + angleForWallPlacement)) {
-
-            this.lookPoint.copy(reticle.position).add(this.upVec);
-            this.lookPoint.y = reticle.position.y;
-            reticle.lookAt(this.lookPoint);
-        }
-        else {
-            camWP.y = reticleWp.y;
-            reticle.lookAt(camWP);
-        }
-
-        // TODO: ability to scale the reticle so that we can fit the scene depending on the view angle or distance to the reticle.
-        // Currently, doing this leads to wrong placement of the scene.
-        /*
-        const rigScale = NeedleXRSession.active?.rigScale || 1;
-        const scale = distance * rigScale;
-        reticle.scale.set(scale, scale, scale);
-        */
-    }
-
-    private onApplyPose(reticle: Object3D) {
-        const rigObject = NeedleXRSession.active?.rig?.gameObject;
-        if (!rigObject) {
-            console.warn("No rig object to place");
-            return;
-        }
-        // const rigScale = NeedleXRSession.active?.rigScale || 1;
-
-        // save the previous rig parent
-        const previousParent = rigObject.parent || this.context.scene;
-
-        // if we have placed this rig before and this is just "replacing" with the anchor
-        // we need to make sure the XRRig attached to the reticle is at the same position as last time
-        // since in the following code we move it inside the reticle (relative to the reticle)
-        if (this._rigPlacementMatrix) {
-            this._rigPlacementMatrix?.decompose(rigObject.position, rigObject.quaternion, rigObject.scale);
-        }
-        else {
-            this._rigPlacementMatrix = rigObject.matrix.clone();
-        }
-
-        this.applyViewBasedTransform(reticle);
-        reticle.updateMatrix();
-        // attach rig to reticle (since the reticle is in rig space it's a easy way to place the rig where we want it relative to the reticle)
-        this.context.scene.add(reticle);
-        reticle.attach(rigObject);
-        reticle.removeFromParent();
-
-        // move rig now relative to the reticle
-        // TODO support scaled reticle
-        rigObject.scale.set(this.arScale, this.arScale, this.arScale);
-        rigObject.position.multiplyScalar(this.arScale);
-
-        rigObject.updateMatrix();
-        // if invert forward is disabled we need to invert the forward rotation
-        // we want to look into positive Z direction (if invertForward is enabled we look into negative Z direction)
-        if (this.invertForward)
-            rigObject.matrix.premultiply(invertForwardMatrix);
-        rigObject.matrix.premultiply(this._startOffset);
-
-        // apply the rig modifications and add it back to the previous parent
-        rigObject.matrix.decompose(rigObject.position, rigObject.quaternion, rigObject.scale);
-        previousParent.add(rigObject);
-
-    }
-}
-
-
-
-
-class WebXRSessionRootUserInput {
-    private static up = new Vector3(0, 1, 0);
-    private static zero = new Vector3(0, 0, 0);
-    private static one = new Vector3(1, 1, 1);
-
-    oneFingerDrag: boolean = true;
-    twoFingerRotate: boolean = true;
-    twoFingerScale: boolean = true;
-
-    factor: number = 1;
-
-    readonly context: Context;
-    readonly offset: Matrix4;
-    readonly plane: Plane;
-
-    private _scale: number = 1;
-    private _hasChanged: boolean = false;
-
-    get scale() {
-        return this._scale;
-    }
-
-    // readonly translate: Vector3 = new Vector3();
-    // readonly rotation: Quaternion = new Quaternion();
-    // readonly scale: Vector3 = new Vector3(1, 1, 1);
-
-    constructor(context: Context) {
-        this.context = context;
-        this.offset = new Matrix4()
-        this.plane = new Plane();
-        this.plane.setFromNormalAndCoplanarPoint(WebXRSessionRootUserInput.up, WebXRSessionRootUserInput.zero);
-    }
-
-    private _enabled: boolean = false;
-
-    reset() {
-        this._scale = 1;
-        this.offset.identity();
-        this._hasChanged = true;
-    }
-    get hasChanged() { return this._hasChanged; }
-
-    /**
-     * Applies the matrix to the offset matrix
-     * @param matrix the matrix to apply the drag offset to
-     * @param invert if true the offset matrix will be inverted before applying it to the matrix and premultiplied 
-     */
-    applyMatrixTo(matrix: Matrix4, invert: boolean) {
-        this._hasChanged = false;
-        if (invert) {
-            this.offset.invert();
-            matrix.premultiply(this.offset);
-        }
-        else
-            matrix.multiply(this.offset);
-        // if (this._needsUpdate)
-        //     this.updateMatrix();
-        // matrix.premultiply(this._rotationMatrix);
-        // matrix.premultiply(this.offset).premultiply(this._rotationMatrix)
-    }
-
-
-    private readonly currentlyUsedPointerIds = new Set<number>();
-    private readonly currentlyUnusedPointerIds = new Set<number>();
-    get isActive() {
-        return this.currentlyUsedPointerIds.size <= 0 && this.currentlyUnusedPointerIds.size > 0;
-    }
-
-    enable() {
-        if (this._enabled) return;
-        this._enabled = true;
-
-        this.context.input.addEventListener("pointerdown", this.onPointerDownEarly, { queue: InputEventQueue.Early });
-        this.context.input.addEventListener("pointerdown", this.onPointerDownLate, { queue: InputEventQueue.Late });
-        this.context.input.addEventListener("pointerup", this.onPointerUpEarly, { queue: InputEventQueue.Early });
-
-        // TODO: refactor the following events to use the input system
-        window.addEventListener('touchstart', this.touchStart, { passive: false });
-        window.addEventListener('touchmove', this.touchMove, { passive: false });
-        window.addEventListener('touchend', this.touchEnd, { passive: false });
-    }
-    disable() {
-        if (!this._enabled) return;
-        this._enabled = false;
-
-        this.context.input.removeEventListener("pointerdown", this.onPointerDownEarly, { queue: InputEventQueue.Early });
-        this.context.input.removeEventListener("pointerdown", this.onPointerDownLate, { queue: InputEventQueue.Late });
-        this.context.input.removeEventListener("pointerup", this.onPointerUpEarly, { queue: InputEventQueue.Early });
-
-        window.removeEventListener('touchstart', this.touchStart);
-        window.removeEventListener('touchmove', this.touchMove);
-        window.removeEventListener('touchend', this.touchEnd);
-    }
-
-    private onPointerDownEarly = (e: NEPointerEvent) => {
-        if (this.isActive) e.stopPropagation();
-    };
-    private onPointerDownLate = (e: NEPointerEvent) => {
-        if (e.used) this.currentlyUsedPointerIds.add(e.pointerId);
-        else if (this.currentlyUsedPointerIds.size <= 0) this.currentlyUnusedPointerIds.add(e.pointerId);
-    };
-    private onPointerUpEarly = (e: NEPointerEvent) => {
-        this.currentlyUsedPointerIds.delete(e.pointerId);
-        this.currentlyUnusedPointerIds.delete(e.pointerId);
-    };
-
-    // private _needsUpdate: boolean = true;
-    // private _rotationMatrix: Matrix4 = new Matrix4();
-    // private updateMatrix() {
-    //     this._needsUpdate = false;
-    //     this._rotationMatrix.makeRotationFromQuaternion(this.rotation);
-    //     this.offset.compose(this.translate, new Quaternion(), this.scale);
-    //     // const rot = this._tempMatrix.makeRotationY(this.angle);
-    //     // this.translate.applyMatrix4(rot);
-    //     // this.offset.elements[12] = this.translate.x;
-    //     // this.offset.elements[13] = this.translate.y;
-    //     // this.offset.elements[14] = this.translate.z;
-    //     // this.offset.premultiply(rot);
-    //     // const s = this.scale;
-    //     // this.offset.premultiply(this._tempMatrix.makeScale(s, s, s));
-    // }
-
-    private prev: Map<number, { ignore: boolean, x: number, z: number, screenx: number, screeny: number }> = new Map();
-    private _didMultitouch: boolean = false;
-
-    private touchStart = (evt: TouchEvent) => {
-        if (evt.defaultPrevented) return;
-
-        // let isValidTouch = true;
-        // isValidTouch = evt.target === this.context.domElement || evt.target === this.context.renderer.domElement;
-        // if (!isValidTouch) {
-        //     return;
-        // }
-
-        for (let i = 0; i < evt.changedTouches.length; i++) {
-            const touch = evt.changedTouches[i];
-            // if a user starts swiping in the top area of the screen
-            // which might be a gesture to open the menu
-            // we ignore it
-            const ignore = DeviceUtilities.isAndroidDevice() && touch.clientY < window.innerHeight * .1;
-            if (!this.prev.has(touch.identifier))
-                this.prev.set(touch.identifier, {
-                    ignore,
-                    x: 0,
-                    z: 0,
-                    screenx: 0,
-                    screeny: 0,
-                });
-            const prev = this.prev.get(touch.identifier);
-            if (prev) {
-                const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
-                prev.x = pos.x;
-                prev.z = pos.z;
-                prev.screenx = touch.clientX;
-                prev.screeny = touch.clientY;
-            }
-        }
-    }
-    private touchEnd = (evt: TouchEvent) => {
-        if (evt.touches.length <= 0) {
-            this._didMultitouch = false;
-        }
-        for (let i = 0; i < evt.changedTouches.length; i++) {
-            const touch = evt.changedTouches[i];
-            this.prev.delete(touch.identifier);
-        }
-    }
-    private touchMove = (evt: TouchEvent) => {
-        if (evt.defaultPrevented) return;
-        if (!this.isActive) return;
-
-        if (evt.touches.length === 1) {
-            // if we had multiple touches before due to e.g. pinching / rotating
-            // and stopping one of the touches, we don't want to move the scene suddenly
-            // this will be resettet when all touches stop
-            if (this._didMultitouch) {
-                return;
-            }
-            const touch = evt.touches[0];
-            const prev = this.prev.get(touch.identifier);
-            if (!prev || prev.ignore) return;
-            const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
-            const dx = pos.x - prev.x;
-            const dy = pos.z - prev.z;
-            if (dx === 0 && dy === 0) return;
-            if (this.oneFingerDrag)
-                this.addMovement(dx, dy);
-            prev.x = pos.x;
-            prev.z = pos.z;
-            prev.screenx = touch.clientX;
-            prev.screeny = touch.clientY;
-            return;
-        }
-        else if (evt.touches.length === 2) {
-            this._didMultitouch = true;
-            const touch1 = evt.touches[0];
-            const touch2 = evt.touches[1];
-            const prev1 = this.prev.get(touch1.identifier);
-            const prev2 = this.prev.get(touch2.identifier);
-            if (!prev1 || !prev2) return;
-
-            if (this.twoFingerRotate) {
-                const angle1 = Math.atan2(touch1.clientY - touch2.clientY, touch1.clientX - touch2.clientX);
-                const lastAngle = Math.atan2(prev1.screeny - prev2.screeny, prev1.screenx - prev2.screenx);
-                const angleDiff = angle1 - lastAngle;
-                if (Math.abs(angleDiff) > 0.001) {
-                    this.addRotation(angleDiff);
-                }
-            }
-
-            if (this.twoFingerScale) {
-                const distx = touch1.clientX - touch2.clientX;
-                const disty = touch1.clientY - touch2.clientY;
-                const dist = Math.sqrt(distx * distx + disty * disty);
-                const lastDistx = prev1.screenx - prev2.screenx;
-                const lastDisty = prev1.screeny - prev2.screeny;
-                const lastDist = Math.sqrt(lastDistx * lastDistx + lastDisty * lastDisty);
-                const distDiff = dist - lastDist;
-                if (Math.abs(distDiff) > 2) {
-                    this.addScale(distDiff)
-                }
-            }
-
-            prev1.screenx = touch1.clientX;
-            prev1.screeny = touch1.clientY;
-            prev2.screenx = touch2.clientX;
-            prev2.screeny = touch2.clientY;
-        }
-    }
-
-    private readonly _raycaster: Raycaster = new Raycaster();
-    private readonly _intersection: Vector3 = new Vector3();
-    private readonly _screenPos: Vector3 = new Vector3();
-
-    private getPositionOnPlane(tx: number, ty: number): Vector3 {
-        const camera = this.context.mainCamera!;
-        this._screenPos.x = (tx / window.innerWidth) * 2 - 1;
-        this._screenPos.y = -(ty / window.innerHeight) * 2 + 1;
-        this._screenPos.z = 1;
-
-        this._screenPos.unproject(camera);
-        this._raycaster.set(camera.position, this._screenPos.sub(camera.position));
-        this._raycaster.ray.intersectPlane(this.plane, this._intersection);
-        return this._intersection;
-    }
-
-    private addMovement(dx: number, dz: number) {
-        // this.translate.x -= dx;
-        // this.translate.z -= dz;
-        // this._needsUpdate = true;
-        // return
-
-        // increase diff if the scene is scaled small
-        dx /= this._scale;
-        dz /= this._scale;
-
-        dx *= this.factor;
-        dz *= this.factor;
-
-        // apply it
-        this.offset.elements[12] += dx;
-        this.offset.elements[14] += dz;
-        if (dx !== 0 || dz !== 0)
-            this._hasChanged = true;
-    };
-
-    private readonly _tempMatrix: Matrix4 = new Matrix4();
-
-    private addScale(diff: number) {
-        diff /= window.innerWidth
-        diff *= -1;
-
-        // this.scale.x *= 1 + diff;
-        // this.scale.y *= 1 + diff;
-        // this.scale.z *= 1 + diff;
-        // this._needsUpdate = true;
-        // return;
-
-
-        // we use this factor to modify the translation factor (in apply movement)
-        this._scale *= 1 + diff;
-        // apply the scale
-        this._tempMatrix.makeScale(1 - diff, 1 - diff, 1 - diff);
-        this.offset.premultiply(this._tempMatrix);
-        if (diff !== 0)
-            this._hasChanged = true;
-    }
-
-
-    private addRotation(rot: number) {
-        rot *= -1;
-        // this.rotation.multiply(new Quaternion().setFromAxisAngle(WebXRSessionRootUserInput.up, rot));
-        // this._needsUpdate = true;
-        // return;
-        this._tempMatrix.makeRotationY(rot);
-        this.offset.premultiply(this._tempMatrix);
-        if (rot !== 0)
-            this._hasChanged = true;
-    }
-}
+import { AxesHelper, DoubleSide, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, Raycaster, RingGeometry, Scene, Vector3 } from "three";
+
+import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
+import { AssetReference } from "../../engine/engine_addressables.js";
+import { Context } from "../../engine/engine_context.js";
+import { destroy, instantiate } from "../../engine/engine_gameobject.js";
+import { InputEventQueue, NEPointerEvent } from "../../engine/engine_input.js";
+import { getBoundingBox, getTempVector } from "../../engine/engine_three_utils.js";
+import type { IComponent, IGameObject } from "../../engine/engine_types.js";
+import { DeviceUtilities, getParam } from "../../engine/engine_utils.js";
+import { NeedleXRController, type NeedleXREventArgs, type NeedleXRHitTestResult, NeedleXRSession } from "../../engine/engine_xr.js";
+import { Behaviour, GameObject } from "../Component.js";
+
+// https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
+
+const debug = getParam("debugwebxr");
+
+const invertForwardMatrix = new Matrix4().makeRotationY(Math.PI);
+
+
+/**
+ * The WebARSessionRoot is the root object for a WebAR session and used to place the scene in AR.  
+ * It is also responsible for scaling the user in AR and optionally creating a XR anchor for the scene placement.  
+ * @example 
+ * ```ts
+ * WebARSessionRoot.onPlaced((args) => {
+ *    console.log("Scene has been placed in AR");
+ * });
+ * ```
+ * 
+ * @category XR
+ * @group Components
+ */
+export class WebARSessionRoot extends Behaviour {
+
+    private static _eventListeners: { [key: string]: Array<(args: { instance: WebARSessionRoot }) => void> } = {};
+    /**
+     * Event that is called when the scene has been placed in AR.
+     * @param cb the callback that is called when the scene has been placed
+     * @returns a function to remove the event listener
+     */
+    static onPlaced(cb: (args: { instance: WebARSessionRoot }) => void) {
+        const event = "placed";
+        if (!this._eventListeners[event]) this._eventListeners[event] = [];
+        this._eventListeners[event].push(cb);
+        return () => {
+            const index = this._eventListeners[event].indexOf(cb);
+            if (index >= 0) this._eventListeners[event].splice(index, 1);
+        }
+    }
+
+    private static _hasPlaced: boolean = false;
+    /**
+     * @returns true if the scene has been placed in AR by the user or automatic placement
+     */
+    static get hasPlaced(): boolean {
+        return this._hasPlaced;
+    }
+
+
+    /** The scale of the user in AR.  
+     * **NOTE**: a large value makes the scene appear smaller  
+     * @default 1
+     */
+    get arScale(): number {
+        return this._arScale;
+    }
+    set arScale(val: number) {
+        this._arScale = Math.max(0.000001, val);
+        this.onSetScale();
+    }
+    private _arScale: number = 1;
+
+    /** When enabled the placed scene forward direction will towards the XRRig 
+     * @deprecated
+     * @default false
+    */
+    invertForward: boolean = false;
+
+    /** When assigned this asset will be loaded and visualize the placement while in AR
+     * @default null
+     */
+    customReticle?: AssetReference;
+
+    /** Enable touch transform to translate, rotate and scale the scene in AR with multitouch
+     * @default true
+     */
+    arTouchTransform: boolean = true;
+
+    /** When enabled the scene will be placed automatically when a point in the real world is found
+     * @default false
+     */
+    autoPlace: boolean = false;
+
+    /** When enabled the scene center will be automatically calculated from the content in the scene */
+    autoCenter: boolean = false;
+
+    /** Experimental: When enabled we will create a XR anchor for the scene placement    
+     * and make sure the scene is at that anchored point during a XR session 
+     * @default false
+     **/
+    useXRAnchor: boolean = false;
+
+    /** true if we're currently placing the scene */
+    private _isPlacing = true;
+
+    /** This is the world matrix of the ar session root when entering webxr
+     * it is applied when the scene has been placed (e.g. if the session root is x:10, z:10 we want this position to be the center of the scene)
+     */
+    private readonly _startOffset: Matrix4 = new Matrix4();
+
+    private _createdPlacementObject: Object3D | null = null;
+    private readonly _reparentedComponents: Array<{ comp: IComponent, originalObject: IGameObject }> = [];
+
+    // move objects into a temporary scene while placing (which is not rendered) so that the components won't be disabled during this process
+    // e.g. we want the avatar to still be updated while placing
+    // another possibly solution would be to ensure from this component that the Rig is *also* not disabled while placing
+    private readonly _placementScene: Scene = new Scene();
+
+    /** the reticles used for placement */
+    private readonly _reticle: IGameObject[] = [];
+    /** needs to be in sync with the reticles */
+    private readonly _hits: XRHitTestResult[] = [];
+
+    private _placementStartTime: number = -1;
+    private _rigPlacementMatrix?: Matrix4;
+    /** if useAnchor is enabled this is the anchor we have created on placing the scene using the placement hit */
+    private _anchor: XRAnchor | null = null;
+    /** user input is used for ar touch transform */
+    private userInput?: WebXRSessionRootUserInput;
+
+    onEnable(): void {
+        this.customReticle?.preload();
+    }
+
+    supportsXR(mode: XRSessionMode): boolean {
+        return mode === "immersive-ar";
+    }
+
+    onEnterXR(_args: NeedleXREventArgs): void {
+        if (debug) console.log("ENTER WEBXR: SessionRoot start...");
+
+        this._anchor = null;
+        WebARSessionRoot._hasPlaced = false;
+
+        // if (_args.xr.session.enabledFeatures?.includes("image-tracking")) {
+        //     console.warn("Image tracking is enabled - will not place scene");
+        //     return;
+        // }
+
+        // save the transform of the session root in the scene to apply it when placing the scene
+        this.gameObject.updateMatrixWorld();
+        this._startOffset.copy(this.gameObject.matrixWorld);
+
+        // create a new root object for the session placement scripts
+        // and move all the children in the scene in a temporary scene that is not rendered
+        const rootObject = new Object3D();
+        this._createdPlacementObject = rootObject;
+        rootObject.name = "AR Session Root";
+        this._placementScene.name = "AR Placement Scene";
+        this._placementScene.children.length = 0;
+        for (let i = this.context.scene.children.length - 1; i >= 0; i--) {
+            const ch = this.context.scene.children[i];
+            this._placementScene.add(ch);
+        }
+        this.context.scene.add(rootObject);
+
+        if (this.autoCenter) {
+            const bounds = getBoundingBox(this._placementScene.children);
+            const center = bounds.getCenter(new Vector3());
+            const size = bounds.getSize(new Vector3());
+            const matrix = new Matrix4();
+            matrix.makeTranslation(center.x, center.y - size.y * .5, center.z);
+            this._startOffset.multiply(matrix);
+        }
+
+        // reparent components
+        // save which gameobject the sessionroot component was previously attached to
+        this._reparentedComponents.length = 0;
+        this._reparentedComponents.push({ comp: this, originalObject: this.gameObject });
+        GameObject.addComponent(rootObject, this);
+        // const webXR = GameObject.findObjectOfType(WebXR2);
+        // if (webXR) {
+        //     this._reparentedComponents.push({ comp: webXR, originalObject: webXR.gameObject });
+        //     GameObject.addComponent(rootObject, webXR);
+        //     const playerSync = GameObject.findObjectOfType(XRFlag);
+        // }
+
+        // recreate the reticle every time we enter AR
+        for (const ret of this._reticle) {
+            destroy(ret);
+        }
+        this._reticle.length = 0;
+        this._isPlacing = true;
+        // we want to receive pointer events EARLY and prevent interaction with other objects while placing by stopping the event propagation
+        this.context.input.addEventListener("pointerup", this.onPlaceScene, { queue: InputEventQueue.Early });
+    }
+    onLeaveXR() {
+        // TODO: WebARSessionRoot doesnt work when we enter passthrough and leave XR without having placed the session!!!
+        this.context.input.removeEventListener("pointerup", this.onPlaceScene, { queue: InputEventQueue.Early });
+        this.onRevertSceneChanges();
+        // this._anchor?.delete();
+        this._anchor = null;
+        WebARSessionRoot._hasPlaced = false;
+        this._rigPlacementMatrix = undefined;
+    }
+    onUpdateXR(args: NeedleXREventArgs): void {
+
+        // disable session placement while images are being tracked
+        if (args.xr.isTrackingImages) {
+            for (const ret of this._reticle)
+                ret.visible = false;
+            return;
+        }
+
+        if (this._isPlacing) {
+            const rigObject = args.xr.rig?.gameObject;
+            // the rig should be parented to the scene while placing
+            // since the camera is always parented to the rig this ensures that the camera is always rendering
+            if (rigObject && rigObject.parent !== this.context.scene) {
+                this.context.scene.add(rigObject);
+            }
+
+            // in pass through mode we want to place the scene using an XR controller
+            let controllersDidHit = false;
+            // when auto placing we just use the user's view
+            if (args.xr.isPassThrough && args.xr.controllers.length > 0 && !this.autoPlace) {
+                for (const ctrl of args.xr.controllers) {
+                    // with this we can only place with the left / first controller right now
+                    // we also only have one reticle... this should probably be refactored a bit so we can have multiple reticles
+                    // and then place at the reticle for which the user clicked the place button
+                    const hit = ctrl.getHitTest();
+                    if (hit) {
+                        controllersDidHit = true;
+                        this.updateReticleAndHits(args.xr, ctrl.index, hit, args.xr.rigScale);
+                    }
+                }
+            }
+            // in screen AR mode we use "camera" hit testing (or when using the simulator where controller hit testing is not supported)
+            if (!controllersDidHit) {
+                const hit = args.xr.getHitTest();
+                if (hit) {
+                    this.updateReticleAndHits(args.xr, 0, hit, args.xr.rigScale);
+                }
+            }
+
+        }
+        else {
+            // Update anchors, if any
+            if (this._anchor && args.xr.referenceSpace) {
+                const pose = args.xr.frame.getPose(this._anchor.anchorSpace, args.xr.referenceSpace);
+                if (pose && this.context.time.frame % 20 === 0) {
+                    // apply the anchor pose to one of the reticles
+                    const converted = args.xr.convertSpace(pose.transform);
+                    const reticle = this._reticle[0];
+                    if (reticle) {
+                        reticle.position.copy(converted.position);
+                        reticle.quaternion.copy(converted.quaternion);
+                        this.onApplyPose(reticle);
+                    }
+                }
+            }
+
+            // Scene has been placed
+            if (this.arTouchTransform) {
+                if (!this.userInput) this.userInput = new WebXRSessionRootUserInput(this.context);
+                this.userInput?.enable();
+            }
+            else this.userInput?.disable();
+            if (this.arTouchTransform && this.userInput?.hasChanged) {
+                if (args.xr.rig) {
+                    const rig = args.xr.rig.gameObject;
+                    this.userInput.applyMatrixTo(rig.matrix, true);
+                    rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
+                    // if the rig is scaled large we want the drag touch to be faster
+                    this.userInput.factor = rig.scale.x;
+                }
+                this.userInput.reset();
+            }
+        }
+    }
+
+    private updateReticleAndHits(_xr: NeedleXRSession, i: number, hit: NeedleXRHitTestResult, scale: number) {
+        // save the hit test
+        this._hits[i] = hit.hit;
+        let reticle = this._reticle[i];
+        if (!reticle) {
+            if (this.customReticle) {
+                if (this.customReticle.asset) {
+                    reticle = instantiate(this.customReticle.asset);
+                }
+                else {
+                    this.customReticle.loadAssetAsync();
+                    return;
+                }
+            }
+            else {
+                reticle = new Mesh(
+                    new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
+                    new MeshBasicMaterial({ side: DoubleSide, depthTest: false, depthWrite: false, transparent: true, opacity: 1, color: 0xeeeeee })
+                ) as any as IGameObject;
+                reticle.name = "AR Placement Reticle";
+            }
+            if (debug) {
+                const axes = new AxesHelper(1);
+                axes.position.y += .01;
+                reticle.add(axes);
+            }
+            this._reticle[i] = reticle;
+            reticle.matrixAutoUpdate = false;
+            reticle.visible = false;
+        }
+        reticle["lastPos"] = reticle["lastPos"] || hit.position.clone();
+        reticle["lastQuat"] = reticle["lastQuat"] || hit.quaternion.clone();
+        // reticle["targetPos"] = reticle["targetPos"] || hit.position.clone();
+        // reticle["targetQuat"] = reticle["targetQuat"] || hit.quaternion.clone();
+
+        // TODO we likely want the reticle itself to be placed _exactly_ and then the visuals being lerped,
+        // Right now this leads to a "rotation glitch" when going from a horizontal to a vertical surface
+        reticle.position.copy(reticle["lastPos"].lerp(hit.position, this.context.time.deltaTime / .1));
+        reticle["lastPos"].copy(reticle.position);
+        reticle.quaternion.copy(reticle["lastQuat"].slerp(hit.quaternion, this.context.time.deltaTime / .05));
+        reticle["lastQuat"].copy(reticle.quaternion);
+
+        // TODO make sure original reticle asset scale is respected, or document it should be uniformly scaled
+        // scale *= this.customReticle?.asset?.scale?.x || 1;
+        reticle.scale.set(scale, scale, scale);
+
+        // if (this.invertForward) {
+        //     reticle.rotateY(Math.PI);
+        // }
+
+        // Workaround: For a custom reticle we apply the view based transform during placement preview
+        // See NE-4161 for context
+        if (this.customReticle)
+            this.applyViewBasedTransform(reticle);
+
+        reticle.updateMatrix();
+        reticle.visible = true;
+        if (reticle.parent !== this.context.scene)
+            this.context.scene.add(reticle);
+
+        if (this._placementStartTime < 0) {
+            this._placementStartTime = this.context.time.realtimeSinceStartup;
+        }
+
+        if (this.autoPlace) {
+            this.upVec.set(0, 1, 0).applyQuaternion(reticle.quaternion);
+            const isUp = this.upVec.dot(getTempVector(0, 1, 0)) > 0.9;
+            if (isUp) {
+                // We want the reticle to be at a suitable spot for a moment before we place the scene (not place it immediately)
+                let autoplace_timer = reticle["autoplace:timer"] || 0;
+                if (autoplace_timer >= 1) {
+                    reticle.visible = false;
+                    this.onPlaceScene(null);
+                }
+                else {
+                    autoplace_timer += this.context.time.deltaTime;
+                    reticle["autoplace:timer"] = autoplace_timer;
+                }
+            }
+            else {
+                reticle["autoplace:timer"] = 0;
+            }
+        }
+    }
+
+    private onPlaceScene = (evt: NEPointerEvent | null) => {
+        if (this._isPlacing == false) return;
+        if (evt?.used) return;
+
+        let reticle: IGameObject | undefined = this._reticle[0];
+
+        if (!reticle) {
+            console.warn("No reticle to place...");
+            return;
+        }
+
+        if (!reticle.visible && !this.autoPlace) {
+            console.warn("Reticle is not visible (can not place)");
+            return;
+        }
+
+        if (NeedleXRSession.active?.isTrackingImages) {
+            console.warn("Scene Placement is disabled while images are being tracked");
+            return;
+        }
+
+        let hit = this._hits[0];
+
+        if (evt && evt.origin instanceof NeedleXRController) {
+            // until we can use hit testing for both controllers and have multple reticles we only allow placement with the first controller
+            const controllerReticle = this._reticle[evt.origin.index];
+            if (controllerReticle) {
+                reticle = controllerReticle;
+                hit = this._hits[evt.origin.index];
+            }
+        }
+
+        // if we place the scene we don't want this event to be propagated to any sub-objects (via the EventSystem) anymore and trigger e.g. a click on objects for the "place tap" event
+        if (evt) {
+            evt.stopImmediatePropagation();
+            evt.stopPropagation();
+            evt.use();
+        }
+
+        this._isPlacing = false;
+        this.context.input.removeEventListener("pointerup", this.onPlaceScene);
+
+        this.onRevertSceneChanges();
+
+        // TODO: we should probably use the non-lerped position and quaternion here
+        reticle.position.copy(reticle["lastPos"]);
+        reticle.quaternion.copy(reticle["lastQuat"]);
+
+        this.onApplyPose(reticle);
+        WebARSessionRoot._hasPlaced = true;
+
+        if (this.useXRAnchor) {
+            this.onCreateAnchor(NeedleXRSession.active!, hit);
+        }
+
+        if (this.context.xr) {
+            for (const ctrl of this.context.xr.controllers) {
+                ctrl.cancelHitTestSource();
+            }
+        }
+    }
+
+    private onSetScale() {
+        if (!WebARSessionRoot._hasPlaced) return;
+        const rig = NeedleXRSession.active?.rig?.gameObject;
+        if (rig) {
+            const currentScale = NeedleXRSession.active?.rigScale || 1;
+            const newScale = (1 / this._arScale) * currentScale;
+            const scaleMatrix = new Matrix4().makeScale(newScale, newScale, newScale).invert();
+            rig.matrix.premultiply(scaleMatrix);
+            rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
+        }
+    }
+
+    private onRevertSceneChanges() {
+        for (const ret of this._reticle) {
+            if (!ret) continue;
+            ret.visible = false;
+            ret?.removeFromParent();
+        }
+        this._reticle.length = 0;
+
+        for (let i = this._placementScene.children.length - 1; i >= 0; i--) {
+            const ch = this._placementScene.children[i];
+            this.context.scene.add(ch);
+        }
+        this._createdPlacementObject?.removeFromParent();
+
+        for (const reparented of this._reparentedComponents) {
+            GameObject.addComponent(reparented.originalObject, reparented.comp);
+        }
+    }
+
+    private async onCreateAnchor(session: NeedleXRSession, hit: XRHitTestResult) {
+        if (hit.createAnchor === undefined) {
+            console.warn("Hit does not support creating an anchor", hit);
+            if (isDevEnvironment()) showBalloonWarning("Hit does not support creating an anchor");
+            return;
+        }
+        else {
+            const anchor = await hit.createAnchor(session.viewerPose!.transform);
+            // make sure the session is still active
+            if (session.running && anchor) {
+                this._anchor = anchor;
+            }
+        }
+    }
+
+    private upVec: Vector3 = new Vector3(0, 1, 0);
+    private lookPoint: Vector3 = new Vector3();
+    private worldUpVec: Vector3 = new Vector3(0, 1, 0);
+
+    private applyViewBasedTransform(reticle: Object3D) {
+
+        // Make reticle face the user to unify the placement experience across devices.
+        // The pose that we're receiving from the hit test varies between devices:
+        // - Quest: currently aligned to the mesh that was hit (depends on room setup), has changed a couple times
+        // - Android WebXR: looking at the camera, but pretty random when on a wall
+        // - Mozilla WebXR Viewer: aligned to the start of the session
+        const camGo = this.context.mainCamera as Object3D as GameObject;
+        const reticleGo = reticle as GameObject;
+        const camWP = camGo.worldPosition;
+        const reticleWp = reticleGo.worldPosition;
+
+        this.upVec.set(0, 1, 0).applyQuaternion(reticle.quaternion);
+
+        // upVec may be pointing AWAY from us, we have to flip it if that's the case
+        const camPos = camGo.worldPosition;
+        if (camPos) {
+            const camToReticle = reticle.position.clone().sub(camPos);
+            const angle = camToReticle.angleTo(this.upVec);
+            if (angle < Math.PI / 2) {
+                this.upVec.negate();
+            }
+        }
+
+        const upAngle = this.upVec.angleTo(this.worldUpVec) * 180 / Math.PI;
+        // For debugging look angle for AR placement
+        // Gizmos.DrawDirection(reticle.position, upVec, "blue", 0.1);
+        // Gizmos.DrawLabel(reticle.position, upAngle.toFixed(2), 0.1);
+
+        const angleForWallPlacement = 30;
+        if ((upAngle > angleForWallPlacement && upAngle < 180 - angleForWallPlacement) ||
+            (upAngle < -angleForWallPlacement && upAngle > -180 + angleForWallPlacement)) {
+
+            this.lookPoint.copy(reticle.position).add(this.upVec);
+            this.lookPoint.y = reticle.position.y;
+            reticle.lookAt(this.lookPoint);
+        }
+        else {
+            camWP.y = reticleWp.y;
+            reticle.lookAt(camWP);
+        }
+
+        // TODO: ability to scale the reticle so that we can fit the scene depending on the view angle or distance to the reticle.
+        // Currently, doing this leads to wrong placement of the scene.
+        /*
+        const rigScale = NeedleXRSession.active?.rigScale || 1;
+        const scale = distance * rigScale;
+        reticle.scale.set(scale, scale, scale);
+        */
+    }
+
+    private onApplyPose(reticle: Object3D) {
+        const rigObject = NeedleXRSession.active?.rig?.gameObject;
+        if (!rigObject) {
+            console.warn("No rig object to place");
+            return;
+        }
+        // const rigScale = NeedleXRSession.active?.rigScale || 1;
+
+        // save the previous rig parent
+        const previousParent = rigObject.parent || this.context.scene;
+
+        // if we have placed this rig before and this is just "replacing" with the anchor
+        // we need to make sure the XRRig attached to the reticle is at the same position as last time
+        // since in the following code we move it inside the reticle (relative to the reticle)
+        if (this._rigPlacementMatrix) {
+            this._rigPlacementMatrix?.decompose(rigObject.position, rigObject.quaternion, rigObject.scale);
+        }
+        else {
+            this._rigPlacementMatrix = rigObject.matrix.clone();
+        }
+
+        this.applyViewBasedTransform(reticle);
+        reticle.updateMatrix();
+        // attach rig to reticle (since the reticle is in rig space it's a easy way to place the rig where we want it relative to the reticle)
+        this.context.scene.add(reticle);
+        reticle.attach(rigObject);
+        reticle.removeFromParent();
+
+        // move rig now relative to the reticle
+        // TODO support scaled reticle
+        rigObject.scale.set(this.arScale, this.arScale, this.arScale);
+        rigObject.position.multiplyScalar(this.arScale);
+
+        rigObject.updateMatrix();
+        // if invert forward is disabled we need to invert the forward rotation
+        // we want to look into positive Z direction (if invertForward is enabled we look into negative Z direction)
+        if (this.invertForward)
+            rigObject.matrix.premultiply(invertForwardMatrix);
+        rigObject.matrix.premultiply(this._startOffset);
+
+        // apply the rig modifications and add it back to the previous parent
+        rigObject.matrix.decompose(rigObject.position, rigObject.quaternion, rigObject.scale);
+        previousParent.add(rigObject);
+
+    }
+}
+
+
+
+
+class WebXRSessionRootUserInput {
+    private static up = new Vector3(0, 1, 0);
+    private static zero = new Vector3(0, 0, 0);
+    private static one = new Vector3(1, 1, 1);
+
+    oneFingerDrag: boolean = true;
+    twoFingerRotate: boolean = true;
+    twoFingerScale: boolean = true;
+
+    factor: number = 1;
+
+    readonly context: Context;
+    readonly offset: Matrix4;
+    readonly plane: Plane;
+
+    private _scale: number = 1;
+    private _hasChanged: boolean = false;
+
+    get scale() {
+        return this._scale;
+    }
+
+    // readonly translate: Vector3 = new Vector3();
+    // readonly rotation: Quaternion = new Quaternion();
+    // readonly scale: Vector3 = new Vector3(1, 1, 1);
+
+    constructor(context: Context) {
+        this.context = context;
+        this.offset = new Matrix4()
+        this.plane = new Plane();
+        this.plane.setFromNormalAndCoplanarPoint(WebXRSessionRootUserInput.up, WebXRSessionRootUserInput.zero);
+    }
+
+    private _enabled: boolean = false;
+
+    reset() {
+        this._scale = 1;
+        this.offset.identity();
+        this._hasChanged = true;
+    }
+    get hasChanged() { return this._hasChanged; }
+
+    /**
+     * Applies the matrix to the offset matrix
+     * @param matrix the matrix to apply the drag offset to
+     * @param invert if true the offset matrix will be inverted before applying it to the matrix and premultiplied 
+     */
+    applyMatrixTo(matrix: Matrix4, invert: boolean) {
+        this._hasChanged = false;
+        if (invert) {
+            this.offset.invert();
+            matrix.premultiply(this.offset);
+        }
+        else
+            matrix.multiply(this.offset);
+        // if (this._needsUpdate)
+        //     this.updateMatrix();
+        // matrix.premultiply(this._rotationMatrix);
+        // matrix.premultiply(this.offset).premultiply(this._rotationMatrix)
+    }
+
+
+    private readonly currentlyUsedPointerIds = new Set<number>();
+    private readonly currentlyUnusedPointerIds = new Set<number>();
+    get isActive() {
+        return this.currentlyUsedPointerIds.size <= 0 && this.currentlyUnusedPointerIds.size > 0;
+    }
+
+    enable() {
+        if (this._enabled) return;
+        this._enabled = true;
+
+        this.context.input.addEventListener("pointerdown", this.onPointerDownEarly, { queue: InputEventQueue.Early });
+        this.context.input.addEventListener("pointerdown", this.onPointerDownLate, { queue: InputEventQueue.Late });
+        this.context.input.addEventListener("pointerup", this.onPointerUpEarly, { queue: InputEventQueue.Early });
+
+        // TODO: refactor the following events to use the input system
+        window.addEventListener('touchstart', this.touchStart, { passive: false });
+        window.addEventListener('touchmove', this.touchMove, { passive: false });
+        window.addEventListener('touchend', this.touchEnd, { passive: false });
+    }
+    disable() {
+        if (!this._enabled) return;
+        this._enabled = false;
+
+        this.context.input.removeEventListener("pointerdown", this.onPointerDownEarly, { queue: InputEventQueue.Early });
+        this.context.input.removeEventListener("pointerdown", this.onPointerDownLate, { queue: InputEventQueue.Late });
+        this.context.input.removeEventListener("pointerup", this.onPointerUpEarly, { queue: InputEventQueue.Early });
+
+        window.removeEventListener('touchstart', this.touchStart);
+        window.removeEventListener('touchmove', this.touchMove);
+        window.removeEventListener('touchend', this.touchEnd);
+    }
+
+    private onPointerDownEarly = (e: NEPointerEvent) => {
+        if (this.isActive) e.stopPropagation();
+    };
+    private onPointerDownLate = (e: NEPointerEvent) => {
+        if (e.used) this.currentlyUsedPointerIds.add(e.pointerId);
+        else if (this.currentlyUsedPointerIds.size <= 0) this.currentlyUnusedPointerIds.add(e.pointerId);
+    };
+    private onPointerUpEarly = (e: NEPointerEvent) => {
+        this.currentlyUsedPointerIds.delete(e.pointerId);
+        this.currentlyUnusedPointerIds.delete(e.pointerId);
+    };
+
+    // private _needsUpdate: boolean = true;
+    // private _rotationMatrix: Matrix4 = new Matrix4();
+    // private updateMatrix() {
+    //     this._needsUpdate = false;
+    //     this._rotationMatrix.makeRotationFromQuaternion(this.rotation);
+    //     this.offset.compose(this.translate, new Quaternion(), this.scale);
+    //     // const rot = this._tempMatrix.makeRotationY(this.angle);
+    //     // this.translate.applyMatrix4(rot);
+    //     // this.offset.elements[12] = this.translate.x;
+    //     // this.offset.elements[13] = this.translate.y;
+    //     // this.offset.elements[14] = this.translate.z;
+    //     // this.offset.premultiply(rot);
+    //     // const s = this.scale;
+    //     // this.offset.premultiply(this._tempMatrix.makeScale(s, s, s));
+    // }
+
+    private prev: Map<number, { ignore: boolean, x: number, z: number, screenx: number, screeny: number }> = new Map();
+    private _didMultitouch: boolean = false;
+
+    private touchStart = (evt: TouchEvent) => {
+        if (evt.defaultPrevented) return;
+
+        // let isValidTouch = true;
+        // isValidTouch = evt.target === this.context.domElement || evt.target === this.context.renderer.domElement;
+        // if (!isValidTouch) {
+        //     return;
+        // }
+
+        for (let i = 0; i < evt.changedTouches.length; i++) {
+            const touch = evt.changedTouches[i];
+            // if a user starts swiping in the top area of the screen
+            // which might be a gesture to open the menu
+            // we ignore it
+            const ignore = DeviceUtilities.isAndroidDevice() && touch.clientY < window.innerHeight * .1;
+            if (!this.prev.has(touch.identifier))
+                this.prev.set(touch.identifier, {
+                    ignore,
+                    x: 0,
+                    z: 0,
+                    screenx: 0,
+                    screeny: 0,
+                });
+            const prev = this.prev.get(touch.identifier);
+            if (prev) {
+                const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
+                prev.x = pos.x;
+                prev.z = pos.z;
+                prev.screenx = touch.clientX;
+                prev.screeny = touch.clientY;
+            }
+        }
+    }
+    private touchEnd = (evt: TouchEvent) => {
+        if (evt.touches.length <= 0) {
+            this._didMultitouch = false;
+        }
+        for (let i = 0; i < evt.changedTouches.length; i++) {
+            const touch = evt.changedTouches[i];
+            this.prev.delete(touch.identifier);
+        }
+    }
+    private touchMove = (evt: TouchEvent) => {
+        if (evt.defaultPrevented) return;
+        if (!this.isActive) return;
+
+        if (evt.touches.length === 1) {
+            // if we had multiple touches before due to e.g. pinching / rotating
+            // and stopping one of the touches, we don't want to move the scene suddenly
+            // this will be resettet when all touches stop
+            if (this._didMultitouch) {
+                return;
+            }
+            const touch = evt.touches[0];
+            const prev = this.prev.get(touch.identifier);
+            if (!prev || prev.ignore) return;
+            const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
+            const dx = pos.x - prev.x;
+            const dy = pos.z - prev.z;
+            if (dx === 0 && dy === 0) return;
+            if (this.oneFingerDrag)
+                this.addMovement(dx, dy);
+            prev.x = pos.x;
+            prev.z = pos.z;
+            prev.screenx = touch.clientX;
+            prev.screeny = touch.clientY;
+            return;
+        }
+        else if (evt.touches.length === 2) {
+            this._didMultitouch = true;
+            const touch1 = evt.touches[0];
+            const touch2 = evt.touches[1];
+            const prev1 = this.prev.get(touch1.identifier);
+            const prev2 = this.prev.get(touch2.identifier);
+            if (!prev1 || !prev2) return;
+
+            if (this.twoFingerRotate) {
+                const angle1 = Math.atan2(touch1.clientY - touch2.clientY, touch1.clientX - touch2.clientX);
+                const lastAngle = Math.atan2(prev1.screeny - prev2.screeny, prev1.screenx - prev2.screenx);
+                const angleDiff = angle1 - lastAngle;
+                if (Math.abs(angleDiff) > 0.001) {
+                    this.addRotation(angleDiff);
+                }
+            }
+
+            if (this.twoFingerScale) {
+                const distx = touch1.clientX - touch2.clientX;
+                const disty = touch1.clientY - touch2.clientY;
+                const dist = Math.sqrt(distx * distx + disty * disty);
+                const lastDistx = prev1.screenx - prev2.screenx;
+                const lastDisty = prev1.screeny - prev2.screeny;
+                const lastDist = Math.sqrt(lastDistx * lastDistx + lastDisty * lastDisty);
+                const distDiff = dist - lastDist;
+                if (Math.abs(distDiff) > 2) {
+                    this.addScale(distDiff)
+                }
+            }
+
+            prev1.screenx = touch1.clientX;
+            prev1.screeny = touch1.clientY;
+            prev2.screenx = touch2.clientX;
+            prev2.screeny = touch2.clientY;
+        }
+    }
+
+    private readonly _raycaster: Raycaster = new Raycaster();
+    private readonly _intersection: Vector3 = new Vector3();
+    private readonly _screenPos: Vector3 = new Vector3();
+
+    private getPositionOnPlane(tx: number, ty: number): Vector3 {
+        const camera = this.context.mainCamera!;
+        this._screenPos.x = (tx / window.innerWidth) * 2 - 1;
+        this._screenPos.y = -(ty / window.innerHeight) * 2 + 1;
+        this._screenPos.z = 1;
+
+        this._screenPos.unproject(camera);
+        this._raycaster.set(camera.position, this._screenPos.sub(camera.position));
+        this._raycaster.ray.intersectPlane(this.plane, this._intersection);
+        return this._intersection;
+    }
+
+    private addMovement(dx: number, dz: number) {
+        // this.translate.x -= dx;
+        // this.translate.z -= dz;
+        // this._needsUpdate = true;
+        // return
+
+        // increase diff if the scene is scaled small
+        dx /= this._scale;
+        dz /= this._scale;
+
+        dx *= this.factor;
+        dz *= this.factor;
+
+        // apply it
+        this.offset.elements[12] += dx;
+        this.offset.elements[14] += dz;
+        if (dx !== 0 || dz !== 0)
+            this._hasChanged = true;
+    };
+
+    private readonly _tempMatrix: Matrix4 = new Matrix4();
+
+    private addScale(diff: number) {
+        diff /= window.innerWidth
+        diff *= -1;
+
+        // this.scale.x *= 1 + diff;
+        // this.scale.y *= 1 + diff;
+        // this.scale.z *= 1 + diff;
+        // this._needsUpdate = true;
+        // return;
+
+
+        // we use this factor to modify the translation factor (in apply movement)
+        this._scale *= 1 + diff;
+        // apply the scale
+        this._tempMatrix.makeScale(1 - diff, 1 - diff, 1 - diff);
+        this.offset.premultiply(this._tempMatrix);
+        if (diff !== 0)
+            this._hasChanged = true;
+    }
+
+
+    private addRotation(rot: number) {
+        rot *= -1;
+        // this.rotation.multiply(new Quaternion().setFromAxisAngle(WebXRSessionRootUserInput.up, rot));
+        // this._needsUpdate = true;
+        // return;
+        this._tempMatrix.makeRotationY(rot);
+        this.offset.premultiply(this._tempMatrix);
+        if (rot !== 0)
+            this._hasChanged = true;
+    }
+}