@needle-tools/engine 2.49.1-pre → 2.51.0-pre

package/src/engine-components/OrbitControls.ts

@@ -3,13 +3,13 @@ import { Behaviour, GameObject } from "./Component";
 import { OrbitControls as ThreeOrbitControls } from "three/examples/jsm/controls/OrbitControls";
 import { LookAtConstraint } from "./LookAtConstraint";
 import * as THREE from "three";
-import { getWorldPosition } from "../engine/engine_three_utils";
+import { getWorldPosition, setWorldPosition, slerp } from "../engine/engine_three_utils";
 import { RaycastOptions } from "../engine/engine_physics";
 import { serializable } from "../engine/engine_serialization_decorator";
 import { Camera } from "./Camera";
-import { getParam } from "../engine/engine_utils";
+import { getParam, isMobileDevice } from "../engine/engine_utils";
 
-const noLimits = getParam("nocamlimits");
+const freeCam = getParam("freecam");
 
 export class OrbitControls extends Behaviour {
     public get controls() {
@@ -40,7 +40,7 @@ export class OrbitControls extends Behaviour {
     doubleClickToFocus: boolean = true;
 
     // remove once slerp works correctly
-    useSlerp: boolean = false;
+    useSlerp: boolean = true;
 
 
     debugLog: boolean = false;
@@ -76,13 +76,21 @@ export class OrbitControls extends Behaviour {
         }
 
         if (this._controls) {
+            if (freeCam) {
+                this.enablePan = true;
+                this.enableZoom = true;
+                this.middleClickToFocus = true;
+                if(isMobileDevice()) this.doubleClickToFocus = true;
+            }
+
             this._controls.enableDamping = this.enableDamping;
+            //@ts-ignore (not in types)
             this._controls.enableKeys = this.enableKeys;
             this._controls.autoRotate = this.autoRotate;
             this._controls.autoRotateSpeed = this.autoRotateSpeed;
             this._controls.enableZoom = this.enableZoom;
             if (cam?.type === "PerspectiveCamera") {
-                if (noLimits) {
+                if (freeCam) {
                     // dont set limits
                 }
                 else {
@@ -91,7 +99,7 @@ export class OrbitControls extends Behaviour {
                 }
             }
             else {
-                if (noLimits) {
+                if (freeCam) {
                     // dont set limits
                 }
                 else {
@@ -183,14 +191,10 @@ export class OrbitControls extends Behaviour {
 
             // lerp the camera
             if (this._lerpCameraToTarget && this._cameraTargetPosition && this._cameraObject) {
-                if (this.useSlerp && typeof this._cameraObject?.position["slerp"] === "function") {
+                // setWorldPosition(this._cameraObject, this._cameraTargetPosition);
+                if (this.useSlerp) {
                     const position = this._cameraObject?.position;
-                    position
-                        // TODO fix the offset here
-                        // .sub(this._cameraTargetPosition)
-                        //@ts-ignore
-                        .slerp(this._cameraTargetPosition, step)
-                    // .add(this._cameraTargetPosition);
+                    slerp(position, this._cameraTargetPosition, step);
                 }
                 else {
                     this._cameraObject?.position.lerp(this._cameraTargetPosition, step);
@@ -210,7 +214,7 @@ export class OrbitControls extends Behaviour {
             }
         }
 
-        if (this.lookAtConstraint?.locked) this.setFromTargetPosition(0, this.lookAtConstraint01);
+        if (!freeCam && this.lookAtConstraint?.locked) this.setFromTargetPosition(0, this.lookAtConstraint01);
 
 
         if (this._controls && !this.context.isInXR) {
@@ -281,6 +285,7 @@ export class OrbitControls extends Behaviour {
                     this._lerpCameraToTarget = true;
                     const pos = getWorldPosition(this.context.mainCamera);
                     this._cameraTargetPosition = pos.clone().sub(this.controls.target).add(this._lookTargetPosition);
+                    this._cameraObject?.parent?.worldToLocal(this._cameraTargetPosition);
                 }
                 break;
             }

package/src/engine-components/RigidBody.ts

@@ -3,7 +3,7 @@ import * as THREE from 'three'
 import { getWorldPosition } from "../engine/engine_three_utils";
 import { serializable } from "../engine/engine_serialization_decorator";
 import { Watch } from "../engine/engine_utils";
-import { Object3D, Vector3 } from "three";
+import { Matrix4, Object3D, Vector3 } from "three";
 import { IRigidbody } from "../engine/engine_types";
 import { CollisionDetectionMode, RigidbodyConstraints } from "../engine/engine_physics.types";
 import { validate } from "../engine/engine_util_decorator";
@@ -110,6 +110,18 @@ class TransformWatch {
                 this.rotationChanged = true;
             })
         }
+
+        // detect changes in the parent matrix
+        const original = this.obj.matrixWorld.multiplyMatrices.bind(this.obj.matrixWorld);
+        const lastParentMatrix = new Matrix4();
+        this.obj.matrixWorld["multiplyMatrices"] = (parent: Matrix4, matrix: Matrix4) => {
+            if (!lastParentMatrix.equals(parent)) {
+                this.positionChanged = true;
+                this.rotationChanged = true;
+                lastParentMatrix.copy(parent);
+            }
+            return original(parent, matrix);;
+        }
     }
 
     stop() {
@@ -201,22 +213,18 @@ export class Rigidbody extends Behaviour implements IRigidbody {
         else this.constraints &= ~RigidbodyConstraints.FreezeRotationZ;
     }
 
-    private _propertiesChanged: boolean = false;
     private static tempPosition: THREE.Vector3 = new THREE.Vector3();
-    private _currentVelocity!: THREE.Vector3;
-    private _smoothedVelocity!: THREE.Vector3;
-    private _smoothedVelocityGetter!: THREE.Vector3;
-    private _lastPosition!: THREE.Vector3;
+    private _propertiesChanged: boolean = false;
+    private _currentVelocity: THREE.Vector3 = new THREE.Vector3();
+    private _smoothedVelocity: THREE.Vector3 = new THREE.Vector3();
+    private _smoothedVelocityGetter: THREE.Vector3 = new THREE.Vector3();
+    private _lastPosition: THREE.Vector3 = new THREE.Vector3();
 
     private _watch?: TransformWatch;
 
-
     awake() {
         this._watch = undefined;
-        this._currentVelocity = new THREE.Vector3();
-        this._smoothedVelocity = new THREE.Vector3();
-        this._smoothedVelocityGetter = new THREE.Vector3();
-        this._lastPosition = new THREE.Vector3();
+        this._propertiesChanged = false;
     }
 
     onEnable() {

package/src/engine-components/ScreenCapture.ts

@@ -40,6 +40,7 @@ declare type ScreenCaptureOptions = {
 export class ScreenCapture extends Behaviour implements IPointerClickHandler {
 
     onPointerClick() {
+        if(this.context.connection.isInRoom === false) return;
         if (this.isReceiving) {
             if (this.videoPlayer)
                 this.videoPlayer.screenspace = !this.videoPlayer.screenspace;

package/src/engine-components/ShadowCatcher.ts

@@ -45,7 +45,12 @@ export class ShadowCatcher extends Behaviour {
             this.applyMaterialOptions(material);
             material.onBeforeCompile = (shader) => {
                 // see https://github.com/mrdoob/three.js/blob/dev/src/renderers/shaders/ShaderLib/meshphysical.glsl.js#L181
-                shader.fragmentShader = shader.fragmentShader.replace("vec3 outgoingLight = totalDiffuse + totalSpecular + totalEmissiveRadiance;",
+                // see https://github.com/mrdoob/three.js/blob/dev/src/renderers/shaders/ShaderLib.js#LL284C11-L284C11
+                // see https://github.com/mrdoob/three.js/blob/dev/src/renderers/shaders/ShaderLib/shadow.glsl.js#L40
+                // see https://github.com/mrdoob/three.js/blob/dev/src/renderers/shaders/ShaderChunk/shadowmask_pars_fragment.glsl.js#L2
+                // see https://github.com/mrdoob/three.js/blob/dev/src/renderers/shaders/ShaderChunk/shadowmap_pars_fragment.glsl.js#L281
+
+                shader.fragmentShader.replace("vec3 outgoingLight = totalDiffuse + totalSpecular + totalEmissiveRadiance;",
                     `vec3 outgoingLight = totalDiffuse + totalSpecular + totalEmissiveRadiance;
                 
                 // diffuse-only lighting with overdrive to somewhat compensate

package/src/engine-components/Skybox.ts

@@ -2,7 +2,7 @@ import { serializable } from "../engine/engine_serialization_decorator";
 import { Behaviour, GameObject } from "./Component";
 import { RGBELoader } from 'three/examples/jsm/loaders/RGBELoader.js';
 import { EXRLoader } from "three/examples/jsm/loaders/EXRLoader";
-import { EquirectangularRefractionMapping, Texture } from "three"
+import { EquirectangularRefractionMapping, sRGBEncoding, Texture, TextureLoader } from "three"
 import { syncField } from "../engine/engine_networking_auto";
 
 
@@ -21,7 +21,7 @@ export class RemoteSkybox extends Behaviour {
     @serializable()
     environment: boolean = true;
 
-    private _loader?: RGBELoader | EXRLoader;
+    private _loader?: RGBELoader | EXRLoader | TextureLoader;
     private _prevLoadedEnvironment?: Texture;
     private _prevEnvironment: Texture | null = null;
     private _prevBackground: any = null;
@@ -50,13 +50,25 @@ export class RemoteSkybox extends Behaviour {
 
         if (!this._loader) {
             const isEXR = url.endsWith(".exr");
-            this._loader = isEXR ? new EXRLoader() : new RGBELoader();
+            const isHdr = url.endsWith(".hdr");
+            if (isEXR) {
+                this._loader = new EXRLoader();
+            }
+            else if (isHdr) {
+                this._loader = new RGBELoader();
+            }
+            else {
+                this._loader = new TextureLoader();
+            }
         }
         const envMap = await this._loader.loadAsync(url);
         if (!envMap) return;
         if (!this.enabled) return;
         this.url = url;
         envMap.mapping = EquirectangularRefractionMapping;
+        if (this._loader instanceof TextureLoader) {
+            envMap.encoding = sRGBEncoding;
+        }
         this._prevBackground = this.context.scene.background;
         this._prevEnvironment = this.context.scene.environment;
         console.log("Set skybox", this.url);

package/src/engine-components/VideoPlayer.ts

@@ -290,7 +290,8 @@ export class VideoPlayer extends Behaviour {
         this._videoTexture.flipY = false;
         this._videoTexture.encoding = THREE.sRGBEncoding;
         this.handleBeginPlaying(playAutomatically);
-        console.log(this);
+        if(debug)
+            console.log(this);
     }
 
     updateAspect() {

package/src/engine-components/WebARSessionRoot.ts

@@ -1,6 +1,5 @@
 import { Behaviour, GameObject } from "./Component";
-import { XRSession } from 'three'
-import { Matrix4, Object3D, XRPose } from "three";
+import { Matrix4, Object3D } from "three";
 import { WebAR, WebXR } from "./WebXR";
 import { InstancingUtil } from "../engine/engine_instancing";
 import { serializable } from "../engine/engine_serialization_decorator";

package/src/engine-components/WebXR.ts

@@ -1,4 +1,4 @@
-import { ArrayCamera, Color, Euler, EventDispatcher, Group, Matrix4, Mesh, MeshBasicMaterial, Object3D, Quaternion, RingGeometry, Texture, Vector3, XRFrame, XRHitTestSource, XRRigidTransform, XRSession, XRViewerPose } from 'three';
+import { ArrayCamera, Color, Euler, EventDispatcher, Group, Matrix4, Mesh, MeshBasicMaterial, Object3D, Quaternion, RingGeometry, Texture, Vector3 } from 'three';
 import { ARButton } from '../include/three/ARButton.js';
 import { VRButton } from '../include/three/VRButton.js';
 
@@ -7,7 +7,7 @@ import { serializable } from "../engine/engine_serialization_decorator";
 import { XRSessionMode } from "../engine/engine_setup";
 import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
 import { INeedleEngineComponent } from "../engine/engine_types";
-import { getParam, setOrAddParamsToUrl } from "../engine/engine_utils";
+import { getParam, isMozillaXR, setOrAddParamsToUrl } from "../engine/engine_utils";
 
 import { Behaviour, GameObject } from "./Component";
 import { noVoip } from "./Voip";
@@ -19,6 +19,7 @@ import { XRFlag, XRState, XRStateFlag } from "./XRFlag";
 
 
 export async function detectARSupport() {
+    if(isMozillaXR()) return true;
     if ("xr" in navigator) {
         //@ts-ignore
         return (await navigator["xr"].isSessionSupported('immersive-ar')) === true;
@@ -110,8 +111,12 @@ export class WebXR extends Behaviour {
             features.optionalFeatures = ['hit-test', 'dom-overlay'];
         }
         else {
-            console.warn("No dom overlay root found, AR will not work");
+            console.warn("No dom overlay root found, HTML overlays on top of screen-based AR will not work.");
         }
+        features.optionalFeatures.push('plane-detection');
+        // features.optionalFeatures.push('light-estimation');
+        features.optionalFeatures.push('anchors');
+
         const arButton = ARButton.createButton(webXR.context.renderer, features);
         arButton.classList.add('webxr-ar-button');
         arButton.classList.add('webxr-button');
@@ -368,6 +373,7 @@ export class WebXR extends Behaviour {
         // we set layers to sync raycasting and have a similar behaviour to unity
         const xr = this.context.renderer.xr;
         if (this.context.mainCamera) {
+            //@ts-ignore
             const cam = xr.getCamera(this.context.mainCamera) as ArrayCamera;
             for (const c of cam.cameras) {
                 c.layers.enableAll();
@@ -549,7 +555,7 @@ export class WebAR {
         context.scene.background = null;
 
         session.requestReferenceSpace('viewer').then((referenceSpace) => {
-            session.requestHitTestSource({ space: referenceSpace }).then((source) => {
+            session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
                 this.hitTestSource = source;
             }).catch((err) => {
                 this.noHitTestAvailable = true;

package/src/engine-components/WebXRAvatar.ts

@@ -125,7 +125,12 @@ export class WebXRAvatar {
         if (!this.flags)
             return;
         let mask = this.isLocalAvatar ? XRStateFlag.FirstPerson : XRStateFlag.ThirdPerson;
-        // mask |= XRStateFlag.VR;
+        if (this.context.isInVR)
+            mask |= XRStateFlag.VR;
+        else if (this.context.isInAR)
+            mask |= XRStateFlag.AR;
+        else
+            mask |= XRStateFlag.Browser;
         for (const f of this.flags) {
             f.gameObject.visible = true;
             f.UpdateVisible(mask);

package/src/engine-components/WebXRController.ts

@@ -1,4 +1,4 @@
-import { BoxHelper, BufferGeometry, Color, Euler, Group, Intersection, Layers, Line, LineBasicMaterial, Material, Mesh, MeshBasicMaterial, Object3D, PerspectiveCamera, Quaternion, Ray, SphereGeometry, Vector2, Vector3, XRInputSource, XRSession } from "three";
+import { BoxHelper, BufferGeometry, Color, Euler, Group, Intersection, Layers, Line, LineBasicMaterial, Material, Mesh, MeshBasicMaterial, Object3D, PerspectiveCamera, Quaternion, Ray, SphereGeometry, Vector2, Vector3 } from "three";
 import { OculusHandModel } from 'three/examples/jsm/webxr/OculusHandModel.js';
 import { OculusHandPointerModel } from 'three/examples/jsm/webxr/OculusHandPointerModel.js';
 import { XRControllerModel, XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js';
@@ -381,11 +381,12 @@ export class WebXRController extends Behaviour {
         }
 
         if (this.raycastLine) {
+            const allowRaycastLineVisible = this.showRaycastLine && this.context.isInVR;
             if (this.type === ControllerType.Touch) {
                 this.raycastLine.visible = false;
             }
             else if (this.isUsingHands) {
-                this.raycastLine.visible = !this.grabbed && this.showRaycastLine;
+                this.raycastLine.visible = !this.grabbed && allowRaycastLineVisible;
                 setWorldPosition(this.raycastLine, wp);
                 const jnts = this.hand!['joints'];
                 if (jnts) {
@@ -400,7 +401,7 @@ export class WebXRController extends Behaviour {
                 setWorldQuaternion(this.raycastLine, this.rayRotation);
             }
             else {
-                this.raycastLine.visible = this.showRaycastLine;
+                this.raycastLine.visible = allowRaycastLineVisible;
                 setWorldQuaternion(this.raycastLine, this.rayRotation);
                 setWorldPosition(this.raycastLine, wp);
             }

package/src/engine-components/WebXRSync.ts

@@ -1,7 +1,7 @@
 import { Behaviour, GameObject } from "./Component";
 import { RoomEvents, OwnershipModel, NetworkConnection } from "../engine/engine_networking";
 import { WebXR, WebXREvent } from "./WebXR";
-import { Group, Quaternion, Vector3, Vector4, WebXRManager, XRFrame, XRInputSource, XRRigidTransform, XRSession } from "three";
+import { Group, Quaternion, Vector3, Vector4, WebXRManager } from "three";
 import { getParam } from "../engine/engine_utils";
 import { Voip } from "./Voip";
 import { Builder, Long } from "flatbuffers";
@@ -402,6 +402,11 @@ export class WebXRSync extends Behaviour {
     private onXRSessionEnded(_evt: { session: XRSession }) {
         console.log("XR session ended");
         this.context.connection.send(WebXRSyncEvent.WebXR_UserLeft, { id: this.context.connection.connectionId, mode: XRMode.VR });
+        if(this.localAvatar){
+            this.localAvatar?.destroy();
+            this.avatars[this.localAvatar.guid] = undefined;
+            this.localAvatar = null;
+        }
     }
 
     private ownership: OwnershipModel | null = null;

package/src/engine-components/XRFlag.ts

@@ -1,5 +1,7 @@
 import { Behaviour, GameObject } from "./Component";
 import * as utils from "../engine/engine_utils";
+
+
 const debug = utils.getParam("debugflags");
 
 export enum XRStateFlag {
@@ -21,13 +23,12 @@ export class XRState {
     public Mask: XRStateFlag = XRStateFlag.Browser | XRStateFlag.ThirdPerson;
 
     public Has(state: XRStateFlag) {
-
         const res = (this.Mask & state);
-        if (debug) console.log("HAS", state, this.Mask, "Result=" + res);
         return res !== 0;
     }
 
     public Set(state: number) {
+        if(debug) console.warn("Set XR flag state to", state)
         this.Mask = state as number;
         XRFlag.Apply();
     }
@@ -108,23 +109,23 @@ export class XRFlag extends Behaviour {
         const st = state as XRState;
         if (st) {
             if (debug)
-                console.log("use passed in mask")
+                console.warn(this.name, "use passed in mask", st.Mask, this.visibleIn)
             res = st.Has(this.visibleIn);
         }
         else {
             if (debug)
-                console.log("use global mask")
+                console.log(this.name, "use global mask")
             XRState.Global.Has(this.visibleIn);
         }
         if (res === undefined) return;
         if (res) {
             if (debug)
-                console.trace("is visible", this.name, this.gameObject.uuid)
+                console.log(this.name, "is visible", this.gameObject.uuid)
             // this.gameObject.visible = true;
             GameObject.setActive(this.gameObject, true);
         } else {
             if (debug)
-                console.trace("is not visible", this.name, this.gameObject.uuid);
+                console.log(this.name, "is not visible", this.gameObject.uuid);
             const isVisible = this.gameObject.visible;
             if(!isVisible) return;
             this.gameObject.visible = false;

package/src/engine-components/js-extensions/Vector.ts

@@ -1,6 +1,6 @@
 import { applyPrototypeExtensions, registerPrototypeExtensions } from "./ExtensionUtils";
-import { Mathf } from "../../engine/engine_math";
 import { Vector3 } from "three";
+import { slerp } from "../../engine/engine_three_utils";
 
 export function apply(object: Vector3) {
     if (object && object.isVector3 === true) {
@@ -10,10 +10,7 @@ export function apply(object: Vector3) {
 
 
 Vector3.prototype["slerp"] = function (end: Vector3, t: number) {
-    const len1 = this.length();
-    const len2 = end.length();
-    const targetLen = Mathf.lerp(len1, len2, t);
-    return this.lerp(end, t).normalize().multiplyScalar(targetLen);
+    return slerp(this, end, t);
 }
 
 registerPrototypeExtensions(Vector3);

package/src/engine-components/timeline/PlayableDirector.ts

@@ -85,9 +85,6 @@ export class PlayableDirector extends Behaviour {
             console.log(this, this.playableAsset?.tracks);
 
         this.rebuildGraph();
-        if (this.playOnAwake)
-            this.play();
-        else this.evaluate();
 
         if (!this.isValid()) console.warn("PlayableDirector is not valid", this.playableAsset, this.playableAsset?.tracks, Array.isArray(this.playableAsset?.tracks), this);
     }
@@ -230,7 +227,7 @@ export class PlayableDirector extends Behaviour {
     ];
 
     private *internalUpdate() {
-        while (this._isPlaying) {
+        while (this._isPlaying && this.activeAndEnabled) {
             if (!this._isPaused && this._isPlaying) {
                 this._time += this.context.time.deltaTime;
                 this.evaluate();

package/src/engine-components/ui/EventSystem.ts

@@ -18,6 +18,9 @@ const debug = getParam("debugeventsystem");
 
 export class EventSystem extends Behaviour {
 
+
+    private static _eventSystemMap = new Map<Context, EventSystem[]>();
+
     static didSearchEventSystem: boolean = false;
     static createIfNoneExists(context: Context) {
         if (!this.didSearchEventSystem) {
@@ -33,7 +36,15 @@ export class EventSystem extends Behaviour {
         GameObject.addNewComponent(go, EventSystem);
         context.scene.add(go);
     }
-    static systems: EventSystem[] = [];
+    
+    static get systems(): EventSystem[] 
+    {
+        const context = Context.Current;
+        if (!this._eventSystemMap.has(context)) {
+            this._eventSystemMap.set(context, []);
+        }  
+        return this._eventSystemMap.get(context)!;
+    }
 
     //@ts-ignore
     static ensureUpdateMeshUI(instance, context: Context) {