@needle-tools/engine 2.28.0-pre → 2.30.0-pre

package/src/engine-components/SyncedCamera.ts

@@ -12,6 +12,7 @@ import { serializeable } from "../engine/engine_serialization_decorator";
 import { Object3D } from "three";
 import { AvatarMarker } from "./WebXRAvatar";
 import { AssetReference } from "../engine/engine_addressables";
+import { ViewDevice } from "../engine/engine_playerview";
 
 const SyncedCameraModelIdentifier = "SCAM";
 registerType(SyncedCameraModelIdentifier, SyncedCameraModel.getRootAsSyncedCameraModel);
@@ -33,8 +34,7 @@ class CameraModel {
         this.userId = connectionId;
     }
 
-    send(sync: SyncedCamera, con: NetworkConnection) {
-        const cam = sync.cam?.cam;
+    send(cam: THREE.Camera | null | undefined, con: NetworkConnection) {
         if (cam) {
             builder.clear();
             const guid = builder.createString(this.guid);
@@ -53,23 +53,22 @@ class CameraModel {
     }
 }
 
-declare type UserCamInfo =
-    {
-        obj: THREE.Object3D,
-        lastUpdate: number;
-        userId: string;
-    };
+declare type UserCamInfo = {
+    obj: THREE.Object3D,
+    lastUpdate: number;
+    userId: string;
+};
 
 export class SyncedCamera extends Behaviour {
 
-    getUserCamera(userId: string): THREE.Object3D | null {
+    static instances: UserCamInfo[] = [];
+
+    getCameraObject(userId: string): THREE.Object3D | null {
         const guid = this.userToCamMap[userId];
         if (!guid) return null;
         return this.remoteCams[guid].obj;
     }
 
-    @serializeable(Camera)
-    public cam: Camera | null = null;
     @serializeable([Object3D, AssetReference])
     public cameraPrefab: THREE.Object3D | null | AssetReference = null;
 
@@ -97,15 +96,8 @@ export class SyncedCamera extends Behaviour {
             if (this.cameraPrefab && "isObject3D" in this.cameraPrefab) {
                 this.cameraPrefab.visible = false;
             }
-
         }
-    }
 
-    start() {
-        if (!this.cam) {
-            console.warn("Missing camera, fallback to main camera", this);
-            this.cam = this.context.mainCameraComponent as Camera ?? null;
-        }
     }
 
     onEnable(): void {
@@ -129,13 +121,17 @@ export class SyncedCamera extends Behaviour {
                 delete this.remoteCams[guid];
                 if (cam)
                     delete this.userToCamMap[cam.userId];
+
+                SyncedCamera.instances.push(cam);
+                this.context.players.removePlayerView(cam.userId, ViewDevice.Browser);
                 continue;
             }
         }
 
         if (WebXR.IsInWebXR) return;
 
-        if (this.cam === null) {
+        const cam = this.context.mainCamera
+        if (cam === null) {
             this.enabled = false;
             return;
         }
@@ -146,8 +142,8 @@ export class SyncedCamera extends Behaviour {
             this._model = new CameraModel(this.context.connection.connectionId, this.context.connection.connectionId + "_camera");
         }
 
-        const wp = utils.getWorldPosition(this.cam.cam);
-        const wq = utils.getWorldQuaternion(this.cam.cam);
+        const wp = utils.getWorldPosition(cam);
+        const wq = utils.getWorldQuaternion(cam);
         if (wp.distanceTo(this._lastWorldPosition) > 0.001 || wq.angleTo(this._lastWorldQuaternion) > 0.01) {
             this._needsUpdate = true;
         }
@@ -163,7 +159,9 @@ export class SyncedCamera extends Behaviour {
 
         this._lastUpdateTime = this.context.time.realtimeSinceStartup;
         this._needsUpdate = false;
-        this._model.send(this, this.context.connection);
+        this._model.send(cam, this.context.connection);
+        if (!this.context.isInXR)
+            this.context.players.setPlayerView(this.context.connection.connectionId, cam, ViewDevice.Browser);
     }
 
     private onReceivedRemoteCameraInfoBin(model: SyncedCameraModel) {
@@ -183,10 +181,12 @@ export class SyncedCamera extends Behaviour {
                 rc.obj.visible = true;
                 this.gameObject.add(instance);
                 this.userToCamMap[userId] = guid;
+                SyncedCamera.instances.push(rc);
 
                 const marker = GameObject.getOrAddComponent(instance, AvatarMarker);
                 marker.connectionId = userId;
                 marker.avatar = instance;
+
             }
             else {
                 return;
@@ -194,6 +194,7 @@ export class SyncedCamera extends Behaviour {
             // console.log(this.remoteCams);
         }
         const obj = rc.obj;
+        this.context.players.setPlayerView(userId, obj, ViewDevice.Browser);
         rc.lastUpdate = this.context.time.realtimeSinceStartup;
         InstancingUtil.markDirty(obj);
         const pos = model.pos();

package/src/engine-components/SyncedRoom.ts

@@ -44,6 +44,7 @@ export class SyncedRoom extends Behaviour {
     }
 
     tryJoinRoom(call: number = 0): boolean {
+        if(call === undefined) call = 0;
         let hasRoomParameter = false;
         if (this.urlParameterName) {
             const val = utils.getParam(this.urlParameterName);

package/src/engine-components/VideoPlayer.ts

@@ -2,9 +2,10 @@ import { Behaviour } from "./Component";
 import * as THREE from "three";
 import { Material } from "material/Material";
 import { serializeable } from "../engine/engine_serialization_decorator";
-import { Object3D } from "three";
+import { LinearFilter, Object3D, Texture } from "three";
 import { awaitInput } from "../engine/engine_input_utils";
 import { getParam } from "../engine/engine_utils";
+import { Renderer } from "./Renderer";
 
 const debug = getParam("debugvideo");
 
@@ -20,22 +21,17 @@ export enum VideoSource {
 }
 
 export enum VideoAudioOutputMode {
-    /// <summary>
-    ///   <para>Disable the embedded audio.</para>
-    /// </summary>
-    None,
-    /// <summary>
-    ///   <para>Send the embedded audio into a specified AudioSource.</para>
-    /// </summary>
-    AudioSource,
-    /// <summary>
-    ///   <para>Send the embedded audio direct to the platform's audio hardware.</para>
-    /// </summary>
-    Direct,
-    /// <summary>
-    ///   <para>Send the embedded audio to the associated AudioSampleProvider.</para>
-    /// </summary>
-    APIOnly,
+    None = 0,
+    AudioSource = 1,
+    Direct = 2,
+    APIOnly = 3,
+}
+
+export enum VideoRenderMode {
+    CameraFarPlane = 0,
+    CameraNearPlane = 1,
+    RenderTexture = 2,
+    MaterialOverride = 3,
 }
 
 export class VideoPlayer extends Behaviour {
@@ -48,7 +44,16 @@ export class VideoPlayer extends Behaviour {
     playOnEnable?: boolean;
 
     @serializeable()
-    targetMaterialProperty?: string;
+    private renderMode?: VideoRenderMode;
+
+    @serializeable()
+    private targetMaterialProperty?: string;
+
+    @serializeable(Renderer)
+    private targetMaterialRenderer?: Renderer;
+
+    @serializeable(Texture)
+    private targetTexture?: Texture;
 
     @serializeable()
     private time: number = 0;
@@ -250,11 +255,26 @@ export class VideoPlayer extends Behaviour {
         if (!this.enabled) return;
         if (!this.videoElement) return;
 
-        const mat = this.gameObject["material"];
+        let target: Object3D | undefined = this.gameObject;
+
+        switch (this.renderMode) {
+            case VideoRenderMode.MaterialOverride:
+                target = this.targetMaterialRenderer?.gameObject;
+                break;
+            case VideoRenderMode.RenderTexture:
+                console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
+                return;
+        }
+
+        if (!target) {
+            console.error("Missing target for video material renderer", this);
+            return;
+        }
+        const mat = target["material"];
         if (mat) {
             if (mat !== this.videoMaterial) {
                 this.videoMaterial = mat.clone();
-                this.gameObject["material"] = this.videoMaterial;
+                target["material"] = this.videoMaterial;
             }
 
             if (!this.targetMaterialProperty) {
@@ -305,4 +325,60 @@ export class VideoPlayer extends Behaviour {
         this.videoElement.style.visibility = "hidden";
         this.videoElement.style.display = "none";
     }
-}
+}
+
+
+
+
+// class VideoTexture extends Texture {
+
+//     get isVideoTexture() { return true; }
+
+// 	constructor( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy ) {
+
+// 		super( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy );
+
+// 		this.minFilter = minFilter !== undefined ? minFilter : LinearFilter;
+// 		this.magFilter = magFilter !== undefined ? magFilter : LinearFilter;
+
+// 		this.generateMipmaps = false;
+
+// 		const scope = this;
+
+// 		function updateVideo() {
+
+// 			scope.needsUpdate = true;
+// 			video.requestVideoFrameCallback( updateVideo );
+
+// 		}
+
+// 		if ( 'requestVideoFrameCallback' in video ) {
+
+// 			video.requestVideoFrameCallback( updateVideo );
+
+// 		}
+
+// 	}
+
+// 	// clone() {
+
+// 	// 	return new this.constructor( this.image ).copy( this );
+
+// 	// }
+
+// 	update() {
+
+// 		const video = this.image;
+// 		const hasVideoFrameCallback = 'requestVideoFrameCallback' in video;
+
+// 		if ( hasVideoFrameCallback === false && video.readyState >= video.HAVE_CURRENT_DATA ) {
+
+// 			this.needsUpdate = true;
+
+// 		}
+
+// 	}
+
+// }
+
+// export { VideoTexture };

package/src/engine-components/Volume.ts

@@ -2,6 +2,9 @@ import { Behaviour } from "./Component";
 import { NoToneMapping, LinearToneMapping, ACESFilmicToneMapping, ReinhardToneMapping } from "three";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { Context } from "../engine/engine_setup";
+import { getParam } from "../engine/engine_utils";
+
+const debug = getParam("debugvolume");
 
 export enum TonemappingMode {
     None = 0,
@@ -35,21 +38,41 @@ function resolveComponentType(data) {
     return VolumeComponent;
 }
 
+const volumeKey = Symbol("volumeprofile");
+
 export class VolumeProfile {
     @serializeable([d => resolveComponentType(d), VolumeComponent])
     components?: VolumeComponent[];
 
-
     apply(context: Context) {
+        this.onUpdate(context, false);
+    }
+
+    unapply(context: Context) {
+        this.onUpdate(context, true);
+    }
+
+    private onUpdate(context: Context, remove: boolean) {
         if (!this.components) return;
+        const renderer = context.renderer;
+        const currentProfile = renderer[volumeKey];
+        const isActive = currentProfile !== undefined;
+        if (remove) {
+            // can not remove volume profile that is not active
+            if (!isActive) return;
+        }
+        else {
+            renderer[volumeKey] = this;
+        }
         for (const component of this.components) {
 
             if (component instanceof ToneMapping) {
                 const tonemapping = component as ToneMapping;
-                if (!component.active) {
+                if (!component.active || remove) {
                     context.renderer.toneMapping = LinearToneMapping;
                     continue;
                 }
+                if (debug) console.log("VOLUME:", TonemappingMode[tonemapping.mode?.value ?? 0]);
                 switch (tonemapping.mode?.value ?? 0) {
                     case TonemappingMode.None:
                         context.renderer.toneMapping = LinearToneMapping;
@@ -66,9 +89,11 @@ export class VolumeProfile {
                 const colorAdjustments = component as ColorAdjustments;
                 // unity range goes from -15..15
                 // three.js range goes from 0..inf
-                console.log(colorAdjustments.postExposure);
+                if (debug)
+                    console.log(colorAdjustments.postExposure);
                 const exposure = Math.pow(2, colorAdjustments.postExposure?.value ?? 0);
-                context.renderer.toneMappingExposure = colorAdjustments.postExposure?.overrideState ? exposure : 1;
+                const useExposure = colorAdjustments.postExposure?.overrideState && !remove;
+                context.renderer.toneMappingExposure = useExposure ? exposure : 1;
             }
         }
     }
@@ -80,7 +105,25 @@ export class Volume extends Behaviour {
     @serializeable(VolumeProfile)
     sharedProfile?: VolumeProfile;
 
+    awake() {
+        if (debug) {
+            console.log(this);
+            console.log("Press P to toggle post processing");
+            window.addEventListener("keydown", (e) => {
+                if (e.key === "p") {
+                    console.log("Toggle volume: " + this.name, !this.enabled);
+                    this.enabled = !this.enabled;
+                }
+            });
+        }
+    }
+
     onEnable() {
+        if (debug) console.log("APPLY VOLUME", this)
         this.sharedProfile?.apply(this.context);
     }
+
+    onDisable() {
+        this.sharedProfile?.unapply(this.context);
+    }
 }

package/src/engine-components/WebARSessionRoot.ts

@@ -1,7 +1,7 @@
-import { Behaviour } from "./Component";
+import { Behaviour, GameObject } from "./Component";
 import * as THREE from 'three'
-import { Object3D, XRPose } from "three";
-import { WebAR } from "./WebXR";
+import { Matrix4, Object3D, XRPose } from "three";
+import { WebAR, WebXR } from "./WebXR";
 import { InstancingUtil } from "./Renderer";
 import { serializeable } from "../engine/engine_serialization_decorator";
 
@@ -11,6 +11,10 @@ export class WebARSessionRoot extends Behaviour {
 
     webAR: WebAR | null = null;
 
+    get rig(): THREE.Object3D | undefined {
+        return this.webAR?.webxr.Rig;
+    }
+
     @serializeable()
     invertForward: boolean = false;
 
@@ -24,46 +28,57 @@ export class WebARSessionRoot extends Behaviour {
         this.setScale(val);
     }
 
+    private readonly _initalMatrix = new THREE.Matrix4();
+    private readonly _selectStartFn = this.onSelectStart.bind(this);
+    private readonly _selectEndFn = this.onSelectEnd.bind(this);
+
+    start() {
+        const xr = GameObject.findObjectOfType(WebXR);
+        if (xr) {
+            xr.Rig.updateMatrix();
+            this._initalMatrix.copy(xr.Rig.matrix);
+        }
+    }
+
     private _arScale: number = 5;
     private _rig: THREE.Object3D | null = null;
     private _startPose: THREE.Matrix4 | null = null;
     private _placementPose: THREE.Matrix4 | null = null;
     private _isTouching: boolean = false;
+    private _rigStartPose: THREE.Matrix4 | undefined | null = null;
 
     onBegin(session: THREE.XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
         this._startPose = this.gameObject.matrix.clone();
-        session.addEventListener('selectstart', this.onSelectStart.bind(this));
-        session.addEventListener('selectend', this.onSelectEnd.bind(this));
-        setTimeout(() => this.gameObject.visible = false, 1000);
+        this._rigStartPose = this.rig?.matrix.clone();
+        session.addEventListener('selectstart', this._selectStartFn);
+        session.addEventListener('selectend', this._selectEndFn);
+        // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
+
+        // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
+        this.gameObject.visible = false;
+
+        if (this.rig) {
+            // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
+            // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
+            // immersive Ar right now to place the user/offset the session
+            this.rig.matrixAutoUpdate = true;
+            this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
+        }
     }
 
-    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined) {
+    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined): boolean {
 
         if (pose && !this._placementPose) {
             if (this._isTouching) {
                 if (this.webAR) this.webAR.setReticleActive(false);
-                this._placementPose = new THREE.Matrix4().fromArray(pose.transform.matrix).invert();
-                if (rig) {
-
-                    if (this.invertForward) {
-                        const rot = new THREE.Matrix4().makeRotationY(Math.PI);
-                        this._placementPose.premultiply(rot);
-                    }
-                    this._rig = rig;
-
-                    this.setScale(this.arScale);
-                }
-                else this._rig = null;
-                // this.gameObject.matrix.copy(this._placementPose);
-                // if (rig) {
-                //     this.gameObject.matrix.premultiply(rig.matrixWorld)
-                // }
-                this.gameObject.visible = true;
+                this.placeAt(rig, new THREE.Matrix4().fromArray(pose.transform.matrix).invert());
+                return true;
             }
         }
+        return false;
 
         // if (this._placementPose) {
         //     this.gameObject.matrixAutoUpdate = false;
@@ -75,16 +90,47 @@ export class WebARSessionRoot extends Behaviour {
         // }
     }
 
+    placeAt(rig: THREE.Object3D | null, mat: Matrix4) {
+        if (!this._placementPose) this._placementPose = new THREE.Matrix4();
+        this._placementPose.copy(mat);
+        if (rig) {
+
+            if (this.invertForward) {
+                const rot = new THREE.Matrix4().makeRotationY(Math.PI);
+                this._placementPose.premultiply(rot);
+            }
+            this._rig = rig;
+
+            this.setScale(this.arScale);
+        }
+        else this._rig = null;
+        // this.gameObject.matrix.copy(this._placementPose);
+        // if (rig) {
+        //     this.gameObject.matrix.premultiply(rig.matrixWorld)
+        // }
+        this.gameObject.visible = true;
+    }
+
     onEnd(rig: THREE.Object3D | null, _session: THREE.XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
-        if (this._startPose)
+        this.gameObject.matrixAutoUpdate = false;
+        if (this._startPose) {
             this.gameObject.matrix.copy(this._startPose);
-        this.gameObject.matrixAutoUpdate = true;
-        if (rig) rig.matrixAutoUpdate = true;
+        }
+        if (rig) {
+            rig.matrixAutoUpdate = true;
+            if (this._rigStartPose) {
+                this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
+                // console.log(rig.position, rig.quaternion, rig.scale);
+            }
+        }
         InstancingUtil.markDirty(this.gameObject, true);
         // HACK to fix physics being not in correct place after exiting AR
-        setTimeout(() => this.gameObject.visible = true, 100);
+        setTimeout(() => {
+            this.gameObject.matrixAutoUpdate = true;
+            this.gameObject.visible = true;
+        }, 100);
     }
 
 
@@ -101,9 +147,13 @@ export class WebARSessionRoot extends Behaviour {
         if (!rig || !this._placementPose) {
             return;
         }
+        if (!this._rigStartPose) {
+            this._rigStartPose = rig.matrix.clone();
+        }
         rig.matrixAutoUpdate = false;
         rig.matrix.multiplyMatrices(new THREE.Matrix4().makeScale(scale, scale, scale), this._placementPose);
         rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
         rig.updateMatrixWorld();
+        console.log("Place", rig.position);
     }
 }