@needle-tools/engine 2.62.1-pre → 2.63.0-pre

package/src/engine-components/CameraUtils.ts

@@ -1,32 +1,54 @@
 import { OrbitControls } from "./OrbitControls";
-import { Camera } from "./Camera";
 import { addNewComponent } from "../engine/engine_components";
-import { Color, Object3D, Scene, Vector3 } from "three";
-import { ICamera, SourceIdentifier } from "../engine/engine_types";
-import { lookAtInverse } from "../engine/engine_three_utils";
+import { Object3D } from "three";
+import { ICamera } from "../engine/engine_types";
 import { RGBAColor } from "./js-extensions/RGBAColor";
 import { ContextEvent, ContextRegistry } from "../engine/engine_context_registry";
-
+import { getCameraController } from "../engine/engine_camera";
+import { Camera } from "./Camera";
+import { EngineElement } from "../engine/engine_element";
 
 ContextRegistry.registerCallback(ContextEvent.MissingCamera, (evt) => {
-    createCameraWithOrbitControl(evt.context.scene, "unknown");
-});
+    const scene = evt.context.scene;
+    const srcId = "unknown";
 
+    const cameraObject = new Object3D();
+    scene.add(cameraObject);
 
-export function createCameraWithOrbitControl(scene: Scene, source: SourceIdentifier): ICamera {
-    const srcId = source;
-    const go = new Object3D();
-    scene.add(go);
     const camInstance = new Camera();
-    const cam = addNewComponent(go, camInstance, true) as ICamera
+    const cam = addNewComponent(cameraObject, camInstance, true) as ICamera
     cam.sourceId = srcId;
     cam.clearFlags = 2;
     cam.backgroundColor = new RGBAColor(0.5, 0.5, 0.5, 1);
-    const orbit = addNewComponent(go, new OrbitControls(), false) as OrbitControls;
-    orbit.sourceId = srcId;
-    go.position.x = -2;
-    go.position.y = 2;
-    go.position.z = 2;
-    lookAtInverse(go, new Vector3(0, 0, 0));
-    return cam as Camera;
-}
+
+    cameraObject.position.x = -2;
+    cameraObject.position.y = 2;
+    cameraObject.position.z = 2;
+    return cam;
+});
+
+ContextRegistry.registerCallback(ContextEvent.ContextCreated, (evt) => {
+    if (!evt.context.mainCamera) return;
+
+    // check if the <needle-engine controls> is not set to false
+    const engineElement = evt.context.domElement as EngineElement
+
+    if (engineElement?.cameraControls === true) {
+
+        // Check if something else already acts as a camera controller
+        const existing = getCameraController(evt.context.mainCamera);
+        if (existing?.isCameraController === true) {
+            return;
+        }
+
+        const cam = evt.context.mainCameraComponent;
+        const cameraObject = cam?.gameObject;
+        if (cameraObject) {
+            const orbit = addNewComponent(cameraObject, new OrbitControls(), false) as OrbitControls;
+            orbit.sourceId = "unknown";
+        }
+        else {
+            console.warn("Missing camera object, can not add orbit controls")
+        }
+    }
+})

package/src/engine-components/Light.ts

@@ -95,7 +95,17 @@ export class Light extends Behaviour implements ILight {
     public innerSpotAngle: number = 1;
 
     @serializable(Color)
-    public color: THREE.Color = new THREE.Color(0xffffff);
+    set color(val: Color) {
+        this._color = val;
+        if (this.light !== undefined) {
+            this.light.color = val;
+        }
+    }
+    get color(): Color {
+        if (this.light) return this.light.color;
+        return this._color;
+    }
+    public _color: THREE.Color = new THREE.Color(0xffffff);
 
     @serializable()
     set shadowNearPlane(val: number) {
@@ -242,13 +252,14 @@ export class Light extends Behaviour implements ILight {
 
     awake() {
         this.color = new THREE.Color(this.color ?? 0xffffff);
-        if(debug) console.log(this.name, this);
+        if (debug) console.log(this.name, this);
     }
 
     onEnable(): void {
         this.createLight();
         if (this.isBaked) return;
         else if (this.light) {
+            this.light.visible = true;
             if (this.selfIsLight) {
                 // nothing to do
             }
@@ -257,12 +268,14 @@ export class Light extends Behaviour implements ILight {
         }
         if (this.type === LightType.Directional)
             this.startCoroutine(this.updateMainLightRoutine(), FrameEvent.LateUpdate);
-
         this._webXRStartedListener = WebXR.addEventListener(WebXREvent.XRStarted, this.onWebXRStarted.bind(this));
         this._webXREndedListener = WebXR.addEventListener(WebXREvent.XRStopped, this.onWebXREnded.bind(this));
     }
 
     onDisable() {
+        if (this.light) {
+            this.light.visible = false;
+        }
         WebXR.removeEventListener(WebXREvent.XRStarted, this._webXRStartedListener);
         WebXR.removeEventListener(WebXREvent.XRStopped, this._webXREndedListener);
     }

package/src/engine-components/OrbitControls.ts

@@ -6,17 +6,23 @@ import { RaycastOptions } from "../engine/engine_physics";
 import { serializable } from "../engine/engine_serialization_decorator";
 import { getParam, isMobileDevice } from "../engine/engine_utils";
 
-import { Box3, Object3D, PerspectiveCamera, Vector2, Vector3 } from "three";
+import { Camera as ThreeCamera, Box3, Object3D, PerspectiveCamera, Vector2, Vector3 } from "three";
 import { OrbitControls as ThreeOrbitControls } from "three/examples/jsm/controls/OrbitControls";
 import { EventSystem, EventSystemEvents } from "./ui/EventSystem";
-import { transformWithEsbuild } from "vite";
+import { ICameraController } from "../engine/engine_types";
+import { setCameraController } from "../engine/engine_camera";
 
 const freeCam = getParam("freecam");
 
 const disabledKeys = { LEFT: "", UP: "", RIGHT: "", BOTTOM: "" };
 let defaultKeys: any = undefined;
 
-export class OrbitControls extends Behaviour {
+export class OrbitControls extends Behaviour implements ICameraController {
+
+    get isCameraController(): boolean {
+        return true;
+    }
+
     public get controls() {
         return this._controls;
     }
@@ -64,6 +70,7 @@ export class OrbitControls extends Behaviour {
 
     private _eventSystem?: EventSystem;
     private _afterHandleInputFn?: any;
+    private _camera : Camera | null = null;
 
     targetElement: HTMLElement | null = null;
 
@@ -98,8 +105,10 @@ export class OrbitControls extends Behaviour {
 
     onEnable() {
         this._enableTime = this.context.time.time;
-        const camGo = GameObject.getComponent(this.gameObject, Camera);
-        const cam = camGo?.cam;
+        const cameraComponent = GameObject.getComponent(this.gameObject, Camera);
+        this._camera = cameraComponent;
+        const cam = cameraComponent?.cam;
+        if (cam) setCameraController(cam, this, true);
         if (!this._controls) {
             console.assert(cam !== null && cam !== undefined, "Missing camera", this);
             if (cam)
@@ -153,6 +162,9 @@ export class OrbitControls extends Behaviour {
     }
 
     onDisable() {
+        if (this._camera?.cam) {
+            setCameraController(this._camera.cam, this, false);
+        }
         if (this._controls) {
             this._controls.enabled = false;
             this._controls.autoRotate = false;
@@ -160,7 +172,7 @@ export class OrbitControls extends Behaviour {
         }
     }
 
-    private _shouldDisable : boolean = false;
+    private _shouldDisable: boolean = false;
     private afterHandleInput() {
         if (this._controls && this._eventSystem) {
             this._shouldDisable = this._eventSystem.hasActiveUI;

package/src/engine-components/ScreenCapture.ts

@@ -5,7 +5,7 @@ import { Context } from "../engine/engine_setup";
 import { RoomEvents } from "../engine/engine_networking";
 import { UserJoinedOrLeftRoomModel } from "../engine/engine_networking";
 import { serializable } from "../engine/engine_serialization";
-import { IPointerClickHandler } from "./ui/PointerEvents";
+import { IPointerClickHandler, PointerEventData } from "./ui/PointerEvents";
 import { EventDispatcher } from "three";
 import { AudioSource } from "./AudioSource";
 import { getParam } from "../engine/engine_utils";
@@ -39,7 +39,8 @@ declare type ScreenCaptureOptions = {
 
 export class ScreenCapture extends Behaviour implements IPointerClickHandler {
 
-    onPointerClick() {
+    onPointerClick(evt : PointerEventData) {
+        if(evt && evt.pointerId !== 0) return;
         if(this.context.connection.isInRoom === false) return;
         if (this.isReceiving) {
             if (this.videoPlayer)

package/src/engine-components/VideoPlayer.ts

@@ -122,6 +122,10 @@ export class VideoPlayer extends Behaviour {
         }
     }
 
+    get videoMaterial() {
+        return this._videoMaterial;
+    }
+
     get videoTexture() {
         return this._videoTexture;
     }
@@ -140,7 +144,7 @@ export class VideoPlayer extends Behaviour {
 
     private _videoElement: HTMLVideoElement | null = null;
     private _videoTexture: THREE.VideoTexture | null = null;
-    private videoMaterial: Material | null = null;
+    private _videoMaterial: Material | null = null;
 
     private _isPlaying: boolean = false;
     private wasPlaying: boolean = false;
@@ -355,18 +359,18 @@ export class VideoPlayer extends Behaviour {
         if (mat) {
             this._targetObjects.push(target);
 
-            if (mat !== this.videoMaterial) {
-                this.videoMaterial = mat.clone();
-                target["material"] = this.videoMaterial;
+            if (mat !== this._videoMaterial) {
+                this._videoMaterial = mat.clone();
+                target["material"] = this._videoMaterial;
             }
 
             if (!this.targetMaterialProperty) {
-                (this.videoMaterial as any).map = this._videoTexture;
+                (this._videoMaterial as any).map = this._videoTexture;
             }
             else {
                 switch (this.targetMaterialProperty) {
                     default:
-                        (this.videoMaterial as any).map = this._videoTexture;
+                        (this._videoMaterial as any).map = this._videoTexture;
                         break;
                     // doesnt render:
                     // case "emissiveTexture":

package/src/engine-components/timeline/TimelineTracks.ts

@@ -276,7 +276,7 @@ export class AnimationTrackHandler extends TrackHandler {
             const isInTimeRange = time >= model.start && time <= model.end;
             const postExtrapolation: Models.ClipExtrapolation = model.postExtrapolationMode;
             let isActive = isInTimeRange;
-            if (!isActive && !didPostExtrapolate && model.end < time && model.postExtrapolationMode !== Models.ClipExtrapolation.None) {
+            if (!isActive && !didPostExtrapolate && model.end < time && postExtrapolation !== Models.ClipExtrapolation.None) {
                 const nextClip = i < this.clips.length - 1 ? this.models[i + 1] : null;
                 // use post extrapolate if its the last clip of the next clip has not yet started
                 if (!nextClip || nextClip.start > time) {
@@ -284,6 +284,12 @@ export class AnimationTrackHandler extends TrackHandler {
                     didPostExtrapolate = true;
                 }
             }
+            const preExtrapolation: Models.ClipExtrapolation = model.preExtrapolationMode;
+            if (i == 0 && !isActive && !didPostExtrapolate && model.start > time && preExtrapolation !== Models.ClipExtrapolation.None) {
+                isActive = true;
+                if (preExtrapolation !== Models.ClipExtrapolation.Hold)
+                    time += model.start;
+            }
             if (isActive) {
                 // const clip = this.clips[i];
                 let weight = 1;
@@ -316,9 +322,9 @@ export class AnimationTrackHandler extends TrackHandler {
                     }
                 }
 
-                if(model.reversed === true) action.time = action.getClip().duration - t;
+                if (model.reversed === true) action.time = action.getClip().duration - t;
                 else action.time = t;
-                
+
                 action.timeScale = 0;
                 const effectiveWeight = weight * this.director.weight;
                 action.weight = effectiveWeight;

package/src/engine-components/ui/BaseUIComponent.ts

@@ -53,6 +53,13 @@ export class BaseUIComponent extends Behaviour {
     // private _intermediate?: Object3D;
     protected _parentComponent?: BaseUIComponent | null = undefined;
 
+    __internalNewInstanceCreated() {
+        super.__internalNewInstanceCreated();
+        this.shadowComponent = null;
+        this._root = undefined;
+        this._parentComponent = undefined;
+    }
+
     onEnable() {
         super.onEnable();
     }
@@ -79,17 +86,7 @@ export class BaseUIComponent extends Behaviour {
         // this.gameObject.layers.set(2)
 
 
-        // TODO: only traverse our own hierarchy, we can stop if we find another owner
-        container.traverse(c => {
-            if (c[$shadowDomOwner] === undefined) {
-                c[$shadowDomOwner] = this;
-            }
-
-            // // this makes text not render anymore when enabled again
-            // if (raycastTarget === false) {
-            //     c.layers.set(2);
-            // }
-        });
+        this.setShadowComponentOwner(container);
 
         let needsUpdate = false;
 
@@ -136,6 +133,19 @@ export class BaseUIComponent extends Behaviour {
         // })
     }
 
+    protected setShadowComponentOwner(current: Object3D | null | undefined) {
+        if (!current) return;
+        // TODO: only traverse our own hierarchy, we can stop if we find another owner
+        if (current[$shadowDomOwner] === undefined || current[$shadowDomOwner] === this) {
+            current[$shadowDomOwner] = this;
+            if (current.children) {
+                for (const ch of current.children) {
+                    this.setShadowComponentOwner(ch);
+                }
+            }
+        }
+    }
+
     private traverseOwnedShadowComponents(current: Object3D, owner: any, callback: (obj: any) => void) {
         if (!current) return;
         if (current[$shadowDomOwner] === owner) {

package/src/engine-components/ui/Button.ts

@@ -6,6 +6,7 @@ import { RGBAColor } from "../js-extensions/RGBAColor";
 import { serializable } from "../../engine/engine_serialization_decorator";
 import { Animator } from "../Animator";
 import { getParam } from "../../engine/engine_utils";
+import { showBalloonMessage } from "../../engine/debug";
 
 const debug = getParam("debugbutton");
 
@@ -102,7 +103,7 @@ export class Button extends Behaviour implements IPointerEventHandler {
 
     onPointerUp(_) {
         if (debug)
-            console.log("Button Down", this.animationTriggers?.highlightedTrigger, this.animator);
+            console.warn("Button Up", this.animationTriggers?.highlightedTrigger, this.animator, this._isHovered);
         if (this.transition == Transition.Animation && this.animationTriggers && this.animator) {
             this.animator.SetTrigger(this._isHovered ? this.animationTriggers.highlightedTrigger : this.animationTriggers.normalTrigger);
         }
@@ -112,8 +113,10 @@ export class Button extends Behaviour implements IPointerEventHandler {
     }
 
     onPointerClick(_args: PointerEventData) {
-        if (debug)
-            console.trace("Button Click", this.onClick);
+        if (debug) {
+            console.warn("Button Click", this.onClick);
+            showBalloonMessage("CLICKED button " + this.name + " at " + this.context.time.frameCount);
+        }
         this.onClick?.invoke();
     }
 
@@ -153,6 +156,7 @@ export class Button extends Behaviour implements IPointerEventHandler {
         super.awake();
         if (debug)
             console.log(this);
+        this._isInit = false;
         this.init();
     }
 

package/src/engine-components/ui/EventSystem.ts

@@ -13,6 +13,7 @@ import { ICanvasGroup, IGraphic } from "./Interfaces";
 import { getParam } from "../../engine/engine_utils";
 import { UIRaycastUtils } from "./RaycastUtils";
 import { $shadowDomOwner } from "./BaseUIComponent";
+import { showBalloonMessage, showBalloonWarning } from "../../engine/debug";
 
 const debug = getParam("debugeventsystem");
 
@@ -248,6 +249,8 @@ export class EventSystem extends Behaviour {
             args: args,
             hasActiveUI: this.currentActiveMeshUIComponents.length > 0,
         }
+        if(debug && args.isClicked)
+            showBalloonMessage("EventSystem: " +  args.pointerId + " - " + this.context.time.frame + " - Up:" + args.isUp + ", Down:" + args.isDown)
         this.dispatchEvent(new CustomEvent(EventSystemEvents.BeforeHandleInput, { detail: evt }))
         this.handleIntersections(hits, args);
         this.dispatchEvent(new CustomEvent(EventSystemEvents.AfterHandleInput, { detail: evt }))
@@ -261,11 +264,11 @@ export class EventSystem extends Behaviour {
         else return;
 
         if (this.lastPointerEvent.isUp) {
-            for (const obj of this.raisedPointerDownEvents) {
-                if (obj.onPointerUp) {
-                    obj.onPointerUp(this.lastPointerEvent);
-                }
-            }
+            // for (const obj of this.raisedPointerDownEvents) {
+            //     if (obj.onPointerUp) {
+            //         obj.onPointerUp(this.lastPointerEvent);
+            //     }
+            // }
             this.raisedPointerDownEvents.length = 0;
             this.raisedPointerDownObjects.length = 0;
         }
@@ -398,8 +401,8 @@ export class EventSystem extends Behaviour {
             }
         }
 
-        // if (clicked)
-        //     console.log(this.context.time.frame, object);
+        if (clicked && debug)
+            console.log(this.context.time.frame, object);
         this.objectsHoveredThisFrame.push(object);
 
         if (canvasGroup === null || canvasGroup.interactable) {
@@ -488,7 +491,7 @@ export class EventSystem extends Behaviour {
 class MeshUIHelper {
 
     private static lastSelected: THREE.Object3D | null = null;
-    private static lastUpdateFrame: { context: Context, frame: number }[] = [];
+    private static lastUpdateFrame: { context: Context, frame: number, nextUpdate: number }[] = [];
     private static needsUpdate: boolean = false;
 
     static markDirty() {
@@ -496,11 +499,15 @@ class MeshUIHelper {
     }
 
     static update(threeMeshUI: any, context: Context) {
+        const currentFrame = context.time.frameCount;
         for (const lu of this.lastUpdateFrame) {
             if (lu.context === context) {
-                if (context.time.frameCount === lu.frame) return;
-                lu.frame = context.time.frameCount;
-                if (this.needsUpdate || context.time.frameCount < 1) {
+                if (currentFrame === lu.frame) return;
+                lu.frame = currentFrame;
+                let shouldUpdate = this.needsUpdate || currentFrame < 1;
+                if(lu.nextUpdate === context.time.frameCount) shouldUpdate = true;
+                if(this.needsUpdate) lu.nextUpdate = currentFrame + 3;
+                if (shouldUpdate) {
                     if (debug)
                         console.log("Update threemeshui");
                     this.needsUpdate = false;
@@ -509,11 +516,11 @@ class MeshUIHelper {
                 return;
             }
         }
-        this.lastUpdateFrame = [{ context, frame: context.time.frameCount }];
+        this.lastUpdateFrame = [{ context, frame: currentFrame, nextUpdate: currentFrame }];
         threeMeshUI.update();
     }
 
-    static updateState(intersect: THREE.Object3D, selectState: boolean): ThreeMeshUI.Block | null {
+    static updateState(intersect: THREE.Object3D, _selectState: boolean): ThreeMeshUI.Block | null {
         let foundBlock: ThreeMeshUI.Block | null = null;
 
         if (intersect) {
@@ -522,30 +529,10 @@ class MeshUIHelper {
             if (foundBlock && foundBlock !== this.lastSelected) {
                 const interactable = foundBlock["interactable"];
                 if (interactable === false) return null;
-                if (selectState) {
-                    this.lastSelected = foundBlock;
-                    //@ts-ignore
-                    if (foundBlock.states["pressed"])
-                        //@ts-ignore
-                        foundBlock.setState("pressed");
-
-                } else {
-                    //@ts-ignore
-                    if (foundBlock.states["hovered"])
-                        //@ts-ignore
-                        foundBlock.setState("hovered");
-                };
                 this.needsUpdate = true;
             }
         }
 
-        // Update non-targeted buttons state
-        // MeshUIBaseComponent.objectsWithState.forEach((obj) => {
-        //     if ((!intersect || obj !== foundBlock) && obj.isUI) {
-        //         obj.setState('idle');
-        //     };
-        // });
-
         return foundBlock;
     }
 
@@ -558,15 +545,6 @@ class MeshUIHelper {
 
     static resetState(obj: any) {
         if (!obj) return;
-        const interactable = obj["interactable"];
-        if (interactable === false) {
-            if (obj.states["disabled"])
-                obj.setState("disabled");
-        }
-        else if (obj === this.lastSelected && obj.states["selected"]) {
-            obj.setState("selected");
-        }
-        else obj.setState('normal');
         this.needsUpdate = true;
     }
 

package/src/engine-components/ui/Graphic.ts

@@ -51,6 +51,13 @@ export class Graphic extends BaseUIComponent implements IGraphic {
         return this._rect!;
     }
 
+    __internalNewInstanceCreated(): void {
+        super.__internalNewInstanceCreated();
+        this._rect = null;
+        this.uiObject = null;
+        if(this._color) this._color = this._color.clone();
+    }
+
     setState(state: string) {
         this.makePanel();
         if (this.uiObject) {

package/src/engine-components/ui/Interfaces.ts

@@ -1,10 +1,12 @@
+import { IComponent } from "../../engine/engine_types";
+
 export interface ICanvasGroup {
     get isCanvasGroup() : boolean;
     blocksRaycasts: boolean;
     interactable: boolean;
 }
 
-export interface IGraphic {
+export interface IGraphic extends IComponent {
     get isGraphic() : boolean;
     raycastTarget: boolean;
 }

package/src/engine-components/ui/RaycastUtils.ts

@@ -33,7 +33,7 @@ export class UIRaycastUtils {
             if (canvasGroup.interactable === false) return false;
         }
         // handle Graphic Raycast target
-        const graphic = foreachComponent(obj, c => {
+        const graphic : IGraphic | undefined = foreachComponent(obj, c => {
             if ((c as unknown as IGraphic).isGraphic === true) return c;
             return undefined;
         }, false);
@@ -43,6 +43,7 @@ export class UIRaycastUtils {
                 out.graphic = graphic as IGraphic;
         }
         if (graphic?.raycastTarget === false) return false;
+        if (graphic?.layer === 2) return false;
 
         return true;
     }

package/src/engine-components/ui/Text.ts

@@ -160,7 +160,7 @@ export class Text extends Graphic {
     private _text: string = "";
     private _fontSize: number = 12;
     private _textMeshUi: Array<ThreeMeshUI.Text> | null = null;
-    private _textContainer: any = null;
+    private _textContainer: ThreeMeshUI.Block | null = null;
 
     private getTextOpts(): object {
         let fontSize = this.fontSize;
@@ -185,7 +185,16 @@ export class Text extends Graphic {
         this._didHandleTextRenderOnTop = false;
         if (this.uiObject) {
             // @ts-ignore
-            this.uiObject.onAfterUpdate = this.updateWidth.bind(this);
+            this.uiObject.onAfterUpdate = () => {
+                this.updateWidth();
+                // We need to update the shadow owner when the text updates
+                // because once the font has loaded we get new children (a new mesh)
+                // which is the text, it needs to be linked back to this component 
+                // to be properly handled by the EventSystem
+                // since the EventSystem looks for shadow component owners to handle events
+                this.setShadowComponentOwner(this.uiObject);
+                this.markDirty();
+            };
         }
 
         setTimeout(()=> this.markDirty(), 10);

package/src/needle-engine.ts

@@ -3,8 +3,8 @@ makeErrorsVisibleForDevelopment();
 
 import "./engine/engine_element";
 import "./engine/engine_setup";
-// import "./engine/engine_mainloop";
-// import "./engine-components/DomOverlay";
+
+import "./engine-components/CameraUtils"
 
 
 export { GameObject, Behaviour } from "./engine-components/Component";
@@ -50,3 +50,7 @@ if(!globalThis["THREE"]) {
     globalThis["THREE"] = THREE;
 }
 else console.warn("Threejs is already imported");
+
+
+
+import "./engine/engine_license";