@needle-tools/engine 2.29.0-pre → 2.31.0-pre

package/src/engine-components/SpectatorCamera.ts

@@ -170,6 +170,7 @@ export class SpectatorCamera extends Behaviour {
 
     private onXRSessionStart(_evt) {
         if (!this.isSupportedPlatform()) return;
+        if (debug) console.log(this.context.mainCamera);
         if (this.context.mainCamera) {
             this.followSelf();
         }
@@ -188,6 +189,7 @@ export class SpectatorCamera extends Behaviour {
 
     private followSelf() {
         this.target = this.context.players.getPlayerView(this.context.connection.connectionId);
+        if (debug) console.log("Follow self", this.target);
     }
 
     // TODO: only show Spectator cam for DesktopVR;
@@ -219,6 +221,8 @@ export class SpectatorCamera extends Behaviour {
 
         this.setAvatarFlagsBeforeRender();
 
+        const mainCam = this.context.mainCameraComponent;
+
         // these should not be needed if we don't override viewport/scissor
         // renderer.getViewport(this.currentViewport);
         // renderer.getScissor(this.currentScissor);
@@ -231,7 +235,17 @@ export class SpectatorCamera extends Behaviour {
         // renderer.setViewport(left, bottom, width, height);
         // renderer.setScissor(left, bottom, width, height);
         // renderer.setScissorTest(true);
-        renderer.setClearColor(new THREE.Color(1, 1, 1));
+        if (mainCam) {
+            const backgroundColor = mainCam.backgroundColor;
+            if (backgroundColor)
+                renderer.setClearColor(backgroundColor, backgroundColor.alpha);
+            this.cam.backgroundColor = backgroundColor;
+            this.cam.clearFlags = mainCam.clearFlags;
+            this.cam.nearClipPlane = mainCam.nearClipPlane;
+            this.cam.farClipPlane = mainCam.farClipPlane;
+        }
+        else
+            renderer.setClearColor(new THREE.Color(1, 1, 1));
         renderer.setRenderTarget(null); // null: direct to Canvas
         renderer.xr.enabled = false;
         const cam = this.cam?.cam;
@@ -384,8 +398,16 @@ class SpectatorHandler implements ISpectatorHandler {
         if (!target || !this.follow) return;
         switch (mode) {
             case SpectatorMode.FirstPerson:
-                this.follow.followFactor = 20;
-                this.follow.rotateFactor = 20;
+                if (this.view?.viewDevice !== ViewDevice.Browser) {
+                    // soft follow for AR and VR
+                    this.follow.followFactor = 5;
+                    this.follow.rotateFactor = 5;
+                }
+                else {
+                    // snappy follow for desktop
+                    this.follow.followFactor = 50;
+                    this.follow.rotateFactor = 50;
+                }
                 target.position.set(0, 0, 0);
                 break;
             case SpectatorMode.ThirdPerson:
@@ -427,10 +449,11 @@ class SpectatorSelectionController {
             downTime = this.context.time.time;
         });
         this.context.input.addEventListener(InputEvents.PointerUp, _ => {
-            if (this.context.time.time - downTime > 1) {
+            const dt = this.context.time.time - downTime;
+            if (dt > 1) {
                 this.spectator.stopSpectating();
             }
-            else
+            else if (this.context.input.getPointerClicked(0) && dt < .3)
                 this.trySelectObject();
         });
     }

package/src/engine-components/SyncedRoom.ts

@@ -44,6 +44,7 @@ export class SyncedRoom extends Behaviour {
     }
 
     tryJoinRoom(call: number = 0): boolean {
+        if(call === undefined) call = 0;
         let hasRoomParameter = false;
         if (this.urlParameterName) {
             const val = utils.getParam(this.urlParameterName);

package/src/engine-components/VideoPlayer.ts

@@ -2,9 +2,10 @@ import { Behaviour } from "./Component";
 import * as THREE from "three";
 import { Material } from "material/Material";
 import { serializeable } from "../engine/engine_serialization_decorator";
-import { Object3D } from "three";
+import { LinearFilter, Object3D, Texture } from "three";
 import { awaitInput } from "../engine/engine_input_utils";
 import { getParam } from "../engine/engine_utils";
+import { Renderer } from "./Renderer";
 
 const debug = getParam("debugvideo");
 
@@ -20,22 +21,17 @@ export enum VideoSource {
 }
 
 export enum VideoAudioOutputMode {
-    /// <summary>
-    ///   <para>Disable the embedded audio.</para>
-    /// </summary>
-    None,
-    /// <summary>
-    ///   <para>Send the embedded audio into a specified AudioSource.</para>
-    /// </summary>
-    AudioSource,
-    /// <summary>
-    ///   <para>Send the embedded audio direct to the platform's audio hardware.</para>
-    /// </summary>
-    Direct,
-    /// <summary>
-    ///   <para>Send the embedded audio to the associated AudioSampleProvider.</para>
-    /// </summary>
-    APIOnly,
+    None = 0,
+    AudioSource = 1,
+    Direct = 2,
+    APIOnly = 3,
+}
+
+export enum VideoRenderMode {
+    CameraFarPlane = 0,
+    CameraNearPlane = 1,
+    RenderTexture = 2,
+    MaterialOverride = 3,
 }
 
 export class VideoPlayer extends Behaviour {
@@ -48,7 +44,16 @@ export class VideoPlayer extends Behaviour {
     playOnEnable?: boolean;
 
     @serializeable()
-    targetMaterialProperty?: string;
+    private renderMode?: VideoRenderMode;
+
+    @serializeable()
+    private targetMaterialProperty?: string;
+
+    @serializeable(Renderer)
+    private targetMaterialRenderer?: Renderer;
+
+    @serializeable(Texture)
+    private targetTexture?: Texture;
 
     @serializeable()
     private time: number = 0;
@@ -120,13 +125,26 @@ export class VideoPlayer extends Behaviour {
     private _isPlaying: boolean = false;
     private wasPlaying: boolean = false;
 
+    setVideo(video: MediaStream) {
+        this.clip = video;
+        this.source = VideoSource.VideoClip;
+        if (!this.videoElement) this.create(true);
+        else {
+            // TODO: how to prevent interruption error when another video is already playing
+            this.videoElement.srcObject = video;
+            if (this._isPlaying)
+                this.videoElement.play();
+        }
+    }
+
     setClipURL(url: string) {
         if (this.url === url) return;
         // console.log("SET URL", url);
         this.url = url;
         this.source = VideoSource.Url;
         if (debug) console.log("set url", url);
-        if (this.videoElement) {
+        if (!this.videoElement) this.create(true);
+        else {
             this.videoElement.src = url;
             if (this._isPlaying)
                 this.videoElement.play();
@@ -177,7 +195,8 @@ export class VideoPlayer extends Behaviour {
         awaitInput(() => {
             this._receivedInput = true;
             this.updateVideoElementSettings();
-        })
+        });
+        this._targetObjects = [];
     }
 
     play() {
@@ -238,6 +257,12 @@ export class VideoPlayer extends Behaviour {
         this.handleBeginPlaying(playAutomatically);
     }
 
+    private _targetObjects: Array<Object3D>;
+
+    getTargetObjects(): Array<Object3D> {
+        return Array.from(this._targetObjects);
+    }
+
     private createVideoElement(): HTMLVideoElement {
         const video = document.createElement("video") as HTMLVideoElement;
         if (this._crossOrigin)
@@ -250,11 +275,30 @@ export class VideoPlayer extends Behaviour {
         if (!this.enabled) return;
         if (!this.videoElement) return;
 
-        const mat = this.gameObject["material"];
+        this._targetObjects.length = 0;
+
+        let target: Object3D | undefined = this.gameObject;
+
+        switch (this.renderMode) {
+            case VideoRenderMode.MaterialOverride:
+                target = this.targetMaterialRenderer?.gameObject;
+                break;
+            case VideoRenderMode.RenderTexture:
+                console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
+                return;
+        }
+
+        if (!target) {
+            console.error("Missing target for video material renderer", this);
+            return;
+        }
+        const mat = target["material"];
         if (mat) {
+            this._targetObjects.push(target);
+            
             if (mat !== this.videoMaterial) {
                 this.videoMaterial = mat.clone();
-                this.gameObject["material"] = this.videoMaterial;
+                target["material"] = this.videoMaterial;
             }
 
             if (!this.targetMaterialProperty) {
@@ -305,4 +349,60 @@ export class VideoPlayer extends Behaviour {
         this.videoElement.style.visibility = "hidden";
         this.videoElement.style.display = "none";
     }
-}
+}
+
+
+
+
+// class VideoTexture extends Texture {
+
+//     get isVideoTexture() { return true; }
+
+// 	constructor( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy ) {
+
+// 		super( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy );
+
+// 		this.minFilter = minFilter !== undefined ? minFilter : LinearFilter;
+// 		this.magFilter = magFilter !== undefined ? magFilter : LinearFilter;
+
+// 		this.generateMipmaps = false;
+
+// 		const scope = this;
+
+// 		function updateVideo() {
+
+// 			scope.needsUpdate = true;
+// 			video.requestVideoFrameCallback( updateVideo );
+
+// 		}
+
+// 		if ( 'requestVideoFrameCallback' in video ) {
+
+// 			video.requestVideoFrameCallback( updateVideo );
+
+// 		}
+
+// 	}
+
+// 	// clone() {
+
+// 	// 	return new this.constructor( this.image ).copy( this );
+
+// 	// }
+
+// 	update() {
+
+// 		const video = this.image;
+// 		const hasVideoFrameCallback = 'requestVideoFrameCallback' in video;
+
+// 		if ( hasVideoFrameCallback === false && video.readyState >= video.HAVE_CURRENT_DATA ) {
+
+// 			this.needsUpdate = true;
+
+// 		}
+
+// 	}
+
+// }
+
+// export { VideoTexture };

package/src/engine-components/Volume.ts

@@ -2,6 +2,9 @@ import { Behaviour } from "./Component";
 import { NoToneMapping, LinearToneMapping, ACESFilmicToneMapping, ReinhardToneMapping } from "three";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { Context } from "../engine/engine_setup";
+import { getParam } from "../engine/engine_utils";
+
+const debug = getParam("debugvolume");
 
 export enum TonemappingMode {
     None = 0,
@@ -35,21 +38,41 @@ function resolveComponentType(data) {
     return VolumeComponent;
 }
 
+const volumeKey = Symbol("volumeprofile");
+
 export class VolumeProfile {
     @serializeable([d => resolveComponentType(d), VolumeComponent])
     components?: VolumeComponent[];
 
-
     apply(context: Context) {
+        this.onUpdate(context, false);
+    }
+
+    unapply(context: Context) {
+        this.onUpdate(context, true);
+    }
+
+    private onUpdate(context: Context, remove: boolean) {
         if (!this.components) return;
+        const renderer = context.renderer;
+        const currentProfile = renderer[volumeKey];
+        const isActive = currentProfile !== undefined;
+        if (remove) {
+            // can not remove volume profile that is not active
+            if (!isActive) return;
+        }
+        else {
+            renderer[volumeKey] = this;
+        }
         for (const component of this.components) {
 
             if (component instanceof ToneMapping) {
                 const tonemapping = component as ToneMapping;
-                if (!component.active) {
+                if (!component.active || remove) {
                     context.renderer.toneMapping = LinearToneMapping;
                     continue;
                 }
+                if (debug) console.log("VOLUME:", TonemappingMode[tonemapping.mode?.value ?? 0]);
                 switch (tonemapping.mode?.value ?? 0) {
                     case TonemappingMode.None:
                         context.renderer.toneMapping = LinearToneMapping;
@@ -66,9 +89,11 @@ export class VolumeProfile {
                 const colorAdjustments = component as ColorAdjustments;
                 // unity range goes from -15..15
                 // three.js range goes from 0..inf
-                console.log(colorAdjustments.postExposure);
+                if (debug)
+                    console.log(colorAdjustments.postExposure);
                 const exposure = Math.pow(2, colorAdjustments.postExposure?.value ?? 0);
-                context.renderer.toneMappingExposure = colorAdjustments.postExposure?.overrideState ? exposure : 1;
+                const useExposure = colorAdjustments.postExposure?.overrideState && !remove;
+                context.renderer.toneMappingExposure = useExposure ? exposure : 1;
             }
         }
     }
@@ -80,7 +105,25 @@ export class Volume extends Behaviour {
     @serializeable(VolumeProfile)
     sharedProfile?: VolumeProfile;
 
+    awake() {
+        if (debug) {
+            console.log(this);
+            console.log("Press P to toggle post processing");
+            window.addEventListener("keydown", (e) => {
+                if (e.key === "p") {
+                    console.log("Toggle volume: " + this.name, !this.enabled);
+                    this.enabled = !this.enabled;
+                }
+            });
+        }
+    }
+
     onEnable() {
+        if (debug) console.log("APPLY VOLUME", this)
         this.sharedProfile?.apply(this.context);
     }
+
+    onDisable() {
+        this.sharedProfile?.unapply(this.context);
+    }
 }

package/src/engine-components/WebARSessionRoot.ts

@@ -1,7 +1,7 @@
-import { Behaviour } from "./Component";
+import { Behaviour, GameObject } from "./Component";
 import * as THREE from 'three'
-import { Object3D, XRPose } from "three";
-import { WebAR } from "./WebXR";
+import { Matrix4, Object3D, XRPose } from "three";
+import { WebAR, WebXR } from "./WebXR";
 import { InstancingUtil } from "./Renderer";
 import { serializeable } from "../engine/engine_serialization_decorator";
 
@@ -11,6 +11,10 @@ export class WebARSessionRoot extends Behaviour {
 
     webAR: WebAR | null = null;
 
+    get rig(): THREE.Object3D | undefined {
+        return this.webAR?.webxr.Rig;
+    }
+
     @serializeable()
     invertForward: boolean = false;
 
@@ -24,46 +28,57 @@ export class WebARSessionRoot extends Behaviour {
         this.setScale(val);
     }
 
+    private readonly _initalMatrix = new THREE.Matrix4();
+    private readonly _selectStartFn = this.onSelectStart.bind(this);
+    private readonly _selectEndFn = this.onSelectEnd.bind(this);
+
+    start() {
+        const xr = GameObject.findObjectOfType(WebXR);
+        if (xr) {
+            xr.Rig.updateMatrix();
+            this._initalMatrix.copy(xr.Rig.matrix);
+        }
+    }
+
     private _arScale: number = 5;
     private _rig: THREE.Object3D | null = null;
     private _startPose: THREE.Matrix4 | null = null;
     private _placementPose: THREE.Matrix4 | null = null;
     private _isTouching: boolean = false;
+    private _rigStartPose: THREE.Matrix4 | undefined | null = null;
 
     onBegin(session: THREE.XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
         this._startPose = this.gameObject.matrix.clone();
-        session.addEventListener('selectstart', this.onSelectStart.bind(this));
-        session.addEventListener('selectend', this.onSelectEnd.bind(this));
-        setTimeout(() => this.gameObject.visible = false, 1000);
+        this._rigStartPose = this.rig?.matrix.clone();
+        session.addEventListener('selectstart', this._selectStartFn);
+        session.addEventListener('selectend', this._selectEndFn);
+        // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
+
+        // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
+        this.gameObject.visible = false;
+
+        if (this.rig) {
+            // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
+            // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
+            // immersive Ar right now to place the user/offset the session
+            this.rig.matrixAutoUpdate = true;
+            this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
+        }
     }
 
-    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined) {
+    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined): boolean {
 
         if (pose && !this._placementPose) {
             if (this._isTouching) {
                 if (this.webAR) this.webAR.setReticleActive(false);
-                this._placementPose = new THREE.Matrix4().fromArray(pose.transform.matrix).invert();
-                if (rig) {
-
-                    if (this.invertForward) {
-                        const rot = new THREE.Matrix4().makeRotationY(Math.PI);
-                        this._placementPose.premultiply(rot);
-                    }
-                    this._rig = rig;
-
-                    this.setScale(this.arScale);
-                }
-                else this._rig = null;
-                // this.gameObject.matrix.copy(this._placementPose);
-                // if (rig) {
-                //     this.gameObject.matrix.premultiply(rig.matrixWorld)
-                // }
-                this.gameObject.visible = true;
+                this.placeAt(rig, new THREE.Matrix4().fromArray(pose.transform.matrix).invert());
+                return true;
             }
         }
+        return false;
 
         // if (this._placementPose) {
         //     this.gameObject.matrixAutoUpdate = false;
@@ -75,16 +90,47 @@ export class WebARSessionRoot extends Behaviour {
         // }
     }
 
+    placeAt(rig: THREE.Object3D | null, mat: Matrix4) {
+        if (!this._placementPose) this._placementPose = new THREE.Matrix4();
+        this._placementPose.copy(mat);
+        if (rig) {
+
+            if (this.invertForward) {
+                const rot = new THREE.Matrix4().makeRotationY(Math.PI);
+                this._placementPose.premultiply(rot);
+            }
+            this._rig = rig;
+
+            this.setScale(this.arScale);
+        }
+        else this._rig = null;
+        // this.gameObject.matrix.copy(this._placementPose);
+        // if (rig) {
+        //     this.gameObject.matrix.premultiply(rig.matrixWorld)
+        // }
+        this.gameObject.visible = true;
+    }
+
     onEnd(rig: THREE.Object3D | null, _session: THREE.XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
-        if (this._startPose)
+        this.gameObject.matrixAutoUpdate = false;
+        if (this._startPose) {
             this.gameObject.matrix.copy(this._startPose);
-        this.gameObject.matrixAutoUpdate = true;
-        if (rig) rig.matrixAutoUpdate = true;
+        }
+        if (rig) {
+            rig.matrixAutoUpdate = true;
+            if (this._rigStartPose) {
+                this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
+                // console.log(rig.position, rig.quaternion, rig.scale);
+            }
+        }
         InstancingUtil.markDirty(this.gameObject, true);
         // HACK to fix physics being not in correct place after exiting AR
-        setTimeout(() => this.gameObject.visible = true, 100);
+        setTimeout(() => {
+            this.gameObject.matrixAutoUpdate = true;
+            this.gameObject.visible = true;
+        }, 100);
     }
 
 
@@ -101,9 +147,13 @@ export class WebARSessionRoot extends Behaviour {
         if (!rig || !this._placementPose) {
             return;
         }
+        if (!this._rigStartPose) {
+            this._rigStartPose = rig.matrix.clone();
+        }
         rig.matrixAutoUpdate = false;
         rig.matrix.multiplyMatrices(new THREE.Matrix4().makeScale(scale, scale, scale), this._placementPose);
         rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
         rig.updateMatrixWorld();
+        console.log("Place", rig.position);
     }
 }