@needle-tools/engine 2.29.0-pre → 2.30.0-pre

package/src/engine-components/VideoPlayer.ts

@@ -2,9 +2,10 @@ import { Behaviour } from "./Component";
 import * as THREE from "three";
 import { Material } from "material/Material";
 import { serializeable } from "../engine/engine_serialization_decorator";
-import { Object3D } from "three";
+import { LinearFilter, Object3D, Texture } from "three";
 import { awaitInput } from "../engine/engine_input_utils";
 import { getParam } from "../engine/engine_utils";
+import { Renderer } from "./Renderer";
 
 const debug = getParam("debugvideo");
 
@@ -20,22 +21,17 @@ export enum VideoSource {
 }
 
 export enum VideoAudioOutputMode {
-    /// <summary>
-    ///   <para>Disable the embedded audio.</para>
-    /// </summary>
-    None,
-    /// <summary>
-    ///   <para>Send the embedded audio into a specified AudioSource.</para>
-    /// </summary>
-    AudioSource,
-    /// <summary>
-    ///   <para>Send the embedded audio direct to the platform's audio hardware.</para>
-    /// </summary>
-    Direct,
-    /// <summary>
-    ///   <para>Send the embedded audio to the associated AudioSampleProvider.</para>
-    /// </summary>
-    APIOnly,
+    None = 0,
+    AudioSource = 1,
+    Direct = 2,
+    APIOnly = 3,
+}
+
+export enum VideoRenderMode {
+    CameraFarPlane = 0,
+    CameraNearPlane = 1,
+    RenderTexture = 2,
+    MaterialOverride = 3,
 }
 
 export class VideoPlayer extends Behaviour {
@@ -48,7 +44,16 @@ export class VideoPlayer extends Behaviour {
     playOnEnable?: boolean;
 
     @serializeable()
-    targetMaterialProperty?: string;
+    private renderMode?: VideoRenderMode;
+
+    @serializeable()
+    private targetMaterialProperty?: string;
+
+    @serializeable(Renderer)
+    private targetMaterialRenderer?: Renderer;
+
+    @serializeable(Texture)
+    private targetTexture?: Texture;
 
     @serializeable()
     private time: number = 0;
@@ -250,11 +255,26 @@ export class VideoPlayer extends Behaviour {
         if (!this.enabled) return;
         if (!this.videoElement) return;
 
-        const mat = this.gameObject["material"];
+        let target: Object3D | undefined = this.gameObject;
+
+        switch (this.renderMode) {
+            case VideoRenderMode.MaterialOverride:
+                target = this.targetMaterialRenderer?.gameObject;
+                break;
+            case VideoRenderMode.RenderTexture:
+                console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
+                return;
+        }
+
+        if (!target) {
+            console.error("Missing target for video material renderer", this);
+            return;
+        }
+        const mat = target["material"];
         if (mat) {
             if (mat !== this.videoMaterial) {
                 this.videoMaterial = mat.clone();
-                this.gameObject["material"] = this.videoMaterial;
+                target["material"] = this.videoMaterial;
             }
 
             if (!this.targetMaterialProperty) {
@@ -305,4 +325,60 @@ export class VideoPlayer extends Behaviour {
         this.videoElement.style.visibility = "hidden";
         this.videoElement.style.display = "none";
     }
-}
+}
+
+
+
+
+// class VideoTexture extends Texture {
+
+//     get isVideoTexture() { return true; }
+
+// 	constructor( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy ) {
+
+// 		super( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy );
+
+// 		this.minFilter = minFilter !== undefined ? minFilter : LinearFilter;
+// 		this.magFilter = magFilter !== undefined ? magFilter : LinearFilter;
+
+// 		this.generateMipmaps = false;
+
+// 		const scope = this;
+
+// 		function updateVideo() {
+
+// 			scope.needsUpdate = true;
+// 			video.requestVideoFrameCallback( updateVideo );
+
+// 		}
+
+// 		if ( 'requestVideoFrameCallback' in video ) {
+
+// 			video.requestVideoFrameCallback( updateVideo );
+
+// 		}
+
+// 	}
+
+// 	// clone() {
+
+// 	// 	return new this.constructor( this.image ).copy( this );
+
+// 	// }
+
+// 	update() {
+
+// 		const video = this.image;
+// 		const hasVideoFrameCallback = 'requestVideoFrameCallback' in video;
+
+// 		if ( hasVideoFrameCallback === false && video.readyState >= video.HAVE_CURRENT_DATA ) {
+
+// 			this.needsUpdate = true;
+
+// 		}
+
+// 	}
+
+// }
+
+// export { VideoTexture };

package/src/engine-components/Volume.ts

@@ -2,6 +2,9 @@ import { Behaviour } from "./Component";
 import { NoToneMapping, LinearToneMapping, ACESFilmicToneMapping, ReinhardToneMapping } from "three";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { Context } from "../engine/engine_setup";
+import { getParam } from "../engine/engine_utils";
+
+const debug = getParam("debugvolume");
 
 export enum TonemappingMode {
     None = 0,
@@ -35,21 +38,41 @@ function resolveComponentType(data) {
     return VolumeComponent;
 }
 
+const volumeKey = Symbol("volumeprofile");
+
 export class VolumeProfile {
     @serializeable([d => resolveComponentType(d), VolumeComponent])
     components?: VolumeComponent[];
 
-
     apply(context: Context) {
+        this.onUpdate(context, false);
+    }
+
+    unapply(context: Context) {
+        this.onUpdate(context, true);
+    }
+
+    private onUpdate(context: Context, remove: boolean) {
         if (!this.components) return;
+        const renderer = context.renderer;
+        const currentProfile = renderer[volumeKey];
+        const isActive = currentProfile !== undefined;
+        if (remove) {
+            // can not remove volume profile that is not active
+            if (!isActive) return;
+        }
+        else {
+            renderer[volumeKey] = this;
+        }
         for (const component of this.components) {
 
             if (component instanceof ToneMapping) {
                 const tonemapping = component as ToneMapping;
-                if (!component.active) {
+                if (!component.active || remove) {
                     context.renderer.toneMapping = LinearToneMapping;
                     continue;
                 }
+                if (debug) console.log("VOLUME:", TonemappingMode[tonemapping.mode?.value ?? 0]);
                 switch (tonemapping.mode?.value ?? 0) {
                     case TonemappingMode.None:
                         context.renderer.toneMapping = LinearToneMapping;
@@ -66,9 +89,11 @@ export class VolumeProfile {
                 const colorAdjustments = component as ColorAdjustments;
                 // unity range goes from -15..15
                 // three.js range goes from 0..inf
-                console.log(colorAdjustments.postExposure);
+                if (debug)
+                    console.log(colorAdjustments.postExposure);
                 const exposure = Math.pow(2, colorAdjustments.postExposure?.value ?? 0);
-                context.renderer.toneMappingExposure = colorAdjustments.postExposure?.overrideState ? exposure : 1;
+                const useExposure = colorAdjustments.postExposure?.overrideState && !remove;
+                context.renderer.toneMappingExposure = useExposure ? exposure : 1;
             }
         }
     }
@@ -80,7 +105,25 @@ export class Volume extends Behaviour {
     @serializeable(VolumeProfile)
     sharedProfile?: VolumeProfile;
 
+    awake() {
+        if (debug) {
+            console.log(this);
+            console.log("Press P to toggle post processing");
+            window.addEventListener("keydown", (e) => {
+                if (e.key === "p") {
+                    console.log("Toggle volume: " + this.name, !this.enabled);
+                    this.enabled = !this.enabled;
+                }
+            });
+        }
+    }
+
     onEnable() {
+        if (debug) console.log("APPLY VOLUME", this)
         this.sharedProfile?.apply(this.context);
     }
+
+    onDisable() {
+        this.sharedProfile?.unapply(this.context);
+    }
 }

package/src/engine-components/WebARSessionRoot.ts

@@ -1,7 +1,7 @@
-import { Behaviour } from "./Component";
+import { Behaviour, GameObject } from "./Component";
 import * as THREE from 'three'
-import { Object3D, XRPose } from "three";
-import { WebAR } from "./WebXR";
+import { Matrix4, Object3D, XRPose } from "three";
+import { WebAR, WebXR } from "./WebXR";
 import { InstancingUtil } from "./Renderer";
 import { serializeable } from "../engine/engine_serialization_decorator";
 
@@ -11,6 +11,10 @@ export class WebARSessionRoot extends Behaviour {
 
     webAR: WebAR | null = null;
 
+    get rig(): THREE.Object3D | undefined {
+        return this.webAR?.webxr.Rig;
+    }
+
     @serializeable()
     invertForward: boolean = false;
 
@@ -24,46 +28,57 @@ export class WebARSessionRoot extends Behaviour {
         this.setScale(val);
     }
 
+    private readonly _initalMatrix = new THREE.Matrix4();
+    private readonly _selectStartFn = this.onSelectStart.bind(this);
+    private readonly _selectEndFn = this.onSelectEnd.bind(this);
+
+    start() {
+        const xr = GameObject.findObjectOfType(WebXR);
+        if (xr) {
+            xr.Rig.updateMatrix();
+            this._initalMatrix.copy(xr.Rig.matrix);
+        }
+    }
+
     private _arScale: number = 5;
     private _rig: THREE.Object3D | null = null;
     private _startPose: THREE.Matrix4 | null = null;
     private _placementPose: THREE.Matrix4 | null = null;
     private _isTouching: boolean = false;
+    private _rigStartPose: THREE.Matrix4 | undefined | null = null;
 
     onBegin(session: THREE.XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
         this._startPose = this.gameObject.matrix.clone();
-        session.addEventListener('selectstart', this.onSelectStart.bind(this));
-        session.addEventListener('selectend', this.onSelectEnd.bind(this));
-        setTimeout(() => this.gameObject.visible = false, 1000);
+        this._rigStartPose = this.rig?.matrix.clone();
+        session.addEventListener('selectstart', this._selectStartFn);
+        session.addEventListener('selectend', this._selectEndFn);
+        // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
+
+        // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
+        this.gameObject.visible = false;
+
+        if (this.rig) {
+            // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
+            // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
+            // immersive Ar right now to place the user/offset the session
+            this.rig.matrixAutoUpdate = true;
+            this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
+        }
     }
 
-    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined) {
+    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined): boolean {
 
         if (pose && !this._placementPose) {
             if (this._isTouching) {
                 if (this.webAR) this.webAR.setReticleActive(false);
-                this._placementPose = new THREE.Matrix4().fromArray(pose.transform.matrix).invert();
-                if (rig) {
-
-                    if (this.invertForward) {
-                        const rot = new THREE.Matrix4().makeRotationY(Math.PI);
-                        this._placementPose.premultiply(rot);
-                    }
-                    this._rig = rig;
-
-                    this.setScale(this.arScale);
-                }
-                else this._rig = null;
-                // this.gameObject.matrix.copy(this._placementPose);
-                // if (rig) {
-                //     this.gameObject.matrix.premultiply(rig.matrixWorld)
-                // }
-                this.gameObject.visible = true;
+                this.placeAt(rig, new THREE.Matrix4().fromArray(pose.transform.matrix).invert());
+                return true;
             }
         }
+        return false;
 
         // if (this._placementPose) {
         //     this.gameObject.matrixAutoUpdate = false;
@@ -75,16 +90,47 @@ export class WebARSessionRoot extends Behaviour {
         // }
     }
 
+    placeAt(rig: THREE.Object3D | null, mat: Matrix4) {
+        if (!this._placementPose) this._placementPose = new THREE.Matrix4();
+        this._placementPose.copy(mat);
+        if (rig) {
+
+            if (this.invertForward) {
+                const rot = new THREE.Matrix4().makeRotationY(Math.PI);
+                this._placementPose.premultiply(rot);
+            }
+            this._rig = rig;
+
+            this.setScale(this.arScale);
+        }
+        else this._rig = null;
+        // this.gameObject.matrix.copy(this._placementPose);
+        // if (rig) {
+        //     this.gameObject.matrix.premultiply(rig.matrixWorld)
+        // }
+        this.gameObject.visible = true;
+    }
+
     onEnd(rig: THREE.Object3D | null, _session: THREE.XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
-        if (this._startPose)
+        this.gameObject.matrixAutoUpdate = false;
+        if (this._startPose) {
             this.gameObject.matrix.copy(this._startPose);
-        this.gameObject.matrixAutoUpdate = true;
-        if (rig) rig.matrixAutoUpdate = true;
+        }
+        if (rig) {
+            rig.matrixAutoUpdate = true;
+            if (this._rigStartPose) {
+                this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
+                // console.log(rig.position, rig.quaternion, rig.scale);
+            }
+        }
         InstancingUtil.markDirty(this.gameObject, true);
         // HACK to fix physics being not in correct place after exiting AR
-        setTimeout(() => this.gameObject.visible = true, 100);
+        setTimeout(() => {
+            this.gameObject.matrixAutoUpdate = true;
+            this.gameObject.visible = true;
+        }, 100);
     }
 
 
@@ -101,9 +147,13 @@ export class WebARSessionRoot extends Behaviour {
         if (!rig || !this._placementPose) {
             return;
         }
+        if (!this._rigStartPose) {
+            this._rigStartPose = rig.matrix.clone();
+        }
         rig.matrixAutoUpdate = false;
         rig.matrix.multiplyMatrices(new THREE.Matrix4().makeScale(scale, scale, scale), this._placementPose);
         rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
         rig.updateMatrixWorld();
+        console.log("Place", rig.position);
     }
 }

package/src/engine-components/WebXR.ts

@@ -4,7 +4,7 @@ import { ARButton } from '../include/three/ARButton.js';
 import { VRButton } from '../include/three/VRButton.js';
 
 import * as THREE from 'three'
-import { ArrayCamera, XRHitTestSource } from 'three';
+import { ArrayCamera, Matrix4, Vector3, XRHitTestSource } from 'three';
 
 import * as utils from "../engine/engine_utils";
 import { noVoip } from "./Voip";
@@ -17,6 +17,7 @@ import { EngineElement } from "../engine/engine_element";
 import { AssetReference } from "../engine/engine_addressables";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { WebXRSync } from "./WebXRSync";
+import { XRSessionMode } from "../engine/engine_setup";
 
 
 export async function detectARSupport() {
@@ -73,7 +74,7 @@ export class WebXR extends Behaviour {
         this.events.addEventListener(type, listener);
         return listener;
     }
-    public static removeEventListener(type: string, listener: any) : any {
+    public static removeEventListener(type: string, listener: any): any {
         this.events.removeEventListener(type, listener);
         return listener;
     }
@@ -94,10 +95,10 @@ export class WebXR extends Behaviour {
 
     public static createARButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement {
         const domOverlayRoot = webXR.webAR?.getAROverlayContainer();
-        const features: any = { requiredFeatures: ['hit-test'] };
+        const features: any = {};
         if (domOverlayRoot) {
             features.domOverlay = { root: domOverlayRoot };
-            features.optionalFeatures = ['dom-overlay'];
+            features.optionalFeatures = ['hit-test', 'dom-overlay'];
         }
         const arButton = ARButton.createButton(webXR.context.renderer, features);
         arButton.classList.add('webxr-ar-button');
@@ -120,7 +121,8 @@ export class WebXR extends Behaviour {
         if (session) session.end();
     }
 
-    public get Rig(): THREE.Object3D | null {
+    public get Rig(): THREE.Object3D {
+        if(!this.rig) this.ensureRig();
         return this.rig;
     }
 
@@ -256,8 +258,6 @@ export class WebXR extends Behaviour {
         WebXR.events.dispatchEvent({ type: WebXREvent.XRUpdate, frame: frame, xr: this.context.renderer.xr, rig: this.rig });
     }
 
-
-
     private onClickedARButton() {
         if (!this._isInAR) {
             this._requestedAR = true;
@@ -361,10 +361,12 @@ export class WebXR extends Behaviour {
 
         switch (flag) {
             case XRStateFlag.AR:
+                this.context.xrSessionMode = XRSessionMode.ImmersiveAR;
                 this._isInAR = true;
                 this.webAR?.onBegin(session);
                 break;
             case XRStateFlag.VR:
+                this.context.xrSessionMode = XRSessionMode.ImmersiveVR;
                 this._isInVR = true;
                 this.onEnterVR(session);
                 break;
@@ -391,6 +393,7 @@ export class WebXR extends Behaviour {
         this._isInVR = false;
         this._requestedAR = false;
         this._requestedVR = false;
+        this.context.xrSessionMode = undefined;
 
         if (this.xrMirrorWindow) {
             this.xrMirrorWindow.close();
@@ -448,7 +451,10 @@ export class WebXR extends Behaviour {
 // not sure if this should be a behaviour. 
 // for now we dont really need it to go through the usual update loop
 export class WebAR {
-    private webxr: WebXR;
+
+    get webxr(): WebXR { return this._webxr; }
+
+    private _webxr: WebXR;
 
     private reticle: THREE.Object3D | null = null;
     private hitTestSource: XRHitTestSource | null = null;
@@ -468,11 +474,13 @@ export class WebAR {
     private get context() { return this.webxr.context; }
 
     constructor(webxr: WebXR) {
-        this.webxr = webxr;
+        this._webxr = webxr;
     }
 
     private arDomOverlay: EngineElement | null = null;
     private arOverlayElement: HTMLElement | null = null;
+    private noHitTestAvailable: boolean = false;
+    private didPlaceARSessionRoot: boolean = false;
 
     getAROverlayContainer(): HTMLElement | null {
         this.arDomOverlay = this.webxr.context.domElement as EngineElement;
@@ -484,12 +492,14 @@ export class WebAR {
         this.reticleActive = active;
     }
 
-    onBegin(session: THREE.XRSession) {
+    async onBegin(session: THREE.XRSession) {
         const context = this.webxr.context;
         this.reticleActive = true;
+        this.didPlaceARSessionRoot = false;
 
+        const deviceType = navigator.userAgent?.includes("OculusBrowser") ? ControllerType.PhysicalDevice : ControllerType.Touch;
         for (let i = 0; i < 4; i++) {
-            WebXRController.Create(this.webxr, i, this.webxr.gameObject, ControllerType.Touch)
+            WebXRController.Create(this.webxr, i, this.webxr.gameObject, deviceType)
         }
 
         if (!this.sessionRoot || this.sessionRoot.destroyed || !this.sessionRoot.activeAndEnabled)
@@ -502,6 +512,9 @@ export class WebAR {
         session.requestReferenceSpace('viewer').then((referenceSpace) => {
             session.requestHitTestSource({ space: referenceSpace }).then((source) => {
                 this.hitTestSource = source;
+            }).catch((_) => {
+                this.noHitTestAvailable = true;
+                console.warn("WebXR: Hit test not supported");
             });
         });
 
@@ -534,7 +547,7 @@ export class WebAR {
         if (this.arDomOverlay && this.arOverlayElement) {
             this.arDomOverlay.onEnterAR(session, this.arOverlayElement);
         }
-        
+
         this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
     }
 
@@ -552,11 +565,29 @@ export class WebAR {
             this.sessionRoot.onEnd(this.webxr.Rig, _session);
         }
         if (this.arDomOverlay) this.arDomOverlay.onExitAR(_session);
-        
+
         this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
     }
 
     onUpdate(session: THREE.XRSession, frame: THREE.XRFrame) {
+
+        if (this.noHitTestAvailable === true) {
+            if (this.reticle)
+                this.reticle.visible = false;
+            if (!this.didPlaceARSessionRoot) {
+                this.didPlaceARSessionRoot = true;
+                const rig = this.webxr.Rig;
+                const placementMatrix = arPlacementWithoutHitTestMatrix.clone();
+                if (rig) {
+                    const positionFromRig = new Vector3(0, 0, 0).add(rig.position).divideScalar(this.sessionRoot?.arScale ?? 1);
+                    placementMatrix.multiply(new Matrix4().makeTranslation(positionFromRig.x, positionFromRig.y, positionFromRig.z));
+                    // placementMatrix.setPosition(positionFromRig);
+                }
+                this.sessionRoot?.placeAt(rig, placementMatrix);
+            }
+            return;
+        }
+
         if (!this.hitTestSource) return;
         const hitTestResults = frame.getHitTestResults(this.hitTestSource);
         if (hitTestResults.length) {
@@ -564,7 +595,11 @@ export class WebAR {
             const referenceSpace = this.webxr.context.renderer.xr.getReferenceSpace();
             if (referenceSpace) {
                 const pose = hit.getPose(referenceSpace);
-                this.sessionRoot?.onUpdate(this.webxr.Rig, session, pose);
+
+                if (this.sessionRoot) {
+                    const didPlace = this.sessionRoot.onUpdate(this.webxr.Rig, session, pose);
+                    this.didPlaceARSessionRoot = didPlace;
+                }
 
                 if (this.reticle) {
                     this.reticle.visible = this.reticleActive;
@@ -587,4 +622,4 @@ export class WebAR {
     }
 }
 
-
+const arPlacementWithoutHitTestMatrix = new THREE.Matrix4().identity().makeTranslation(0, -0.5, 0);

package/src/engine-components/WebXRAvatar.ts

@@ -11,6 +11,7 @@ import { VRUserState } from "./WebXRSync";
 import { getParam } from "../engine/engine_utils";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { ViewDevice } from "../engine/engine_playerview";
+import { InstancingUtil } from "./Renderer";
 
 export const debug = getParam("debugavatar");
 
@@ -168,6 +169,8 @@ export class WebXRAvatar {
                 // }
                 this.context.players.setPlayerView(state.guid, viewObj, device);
 
+                InstancingUtil.markDirty(this.head);
+
                 this._canInterpolate = true;
                 const ht = this.isLocalAvatar ? this.head : this._headTarget;
                 ht.position.set(state.position.x, state.position.y, state.position.z);
@@ -183,6 +186,7 @@ export class WebXRAvatar {
                     ht.quaternion.multiply(WebXRAvatar.invertRotation);
                     ht.scale.set(state.scale, state.scale, state.scale);
                     ht.scale.multiply(this.handLeftScale);
+                    InstancingUtil.markDirty(this.handLeft);
                 }
 
                 if (this.handRight) {
@@ -192,6 +196,7 @@ export class WebXRAvatar {
                     ht.quaternion.multiply(WebXRAvatar.invertRotation);
                     ht.scale.set(state.scale, state.scale, state.scale);
                     ht.scale.multiply(this.handRightScale);
+                    InstancingUtil.markDirty(this.handRight);
                 }
             }
         }