@needle-tools/engine 5.1.0-canary.fbdfce3 → 5.1.0-experimental.03e8105

package/src/engine-components/timeline/TimelineModels.ts

@@ -42,7 +42,7 @@ export enum ClipExtrapolation {
  * @category Animation and Sequencing
  * @see {@link PlayableDirector} for the main component to control timelines in Needle Engine.
  */
-export declare type TrackModel = {
+export type TrackModel = {
     name: string;
     type: TrackType;
     muted: boolean;
@@ -53,6 +53,10 @@ export declare type TrackModel = {
     volume?: number;
 }
 
+export function isTrackModel(obj:unknown): obj is TrackModel {
+    return typeof obj === "object" && obj !== null && "name" in obj && "type" in obj && "outputs" in obj && "muted" in obj;
+}
+
 declare type Vec3 = { x: number, y: number, z: number };
 declare type Quat = { x: number, y: number, z: number, w: number };
 

package/src/engine-components/timeline/TimelineTracks.ts

@@ -4,13 +4,14 @@ import { isDevEnvironment } from "../../engine/debug/index.js";
 import { Context } from "../../engine/engine_setup.js";
 import type { Constructor } from "../../engine/engine_types.js";
 import { getParam, resolveUrl } from "../../engine/engine_utils.js";
-import { setObjectAnimated } from "../AnimationUtils.js";
 import { Animator } from "../Animator.js"
 import { AudioSource } from "../AudioSource.js";
 import { GameObject } from "../Component.js";
 import type { PlayableDirector } from "./PlayableDirector.js";
 import { SignalReceiver } from "./SignalAsset.js";
 import * as Models from "./TimelineModels.js";
+import type { TimelineBuilder } from "./TimelineBuilder.js";
+import { AnimationUtils } from "../../engine/engine_animation.js";
 
 const debug = getParam("debugtimeline");
 
@@ -18,7 +19,7 @@ const debug = getParam("debugtimeline");
  * A TrackHandler is responsible for evaluating a specific type of timeline track.  
  * A timeline track can be an animation track, audio track, signal track, control track etc and is controlled by a {@link PlayableDirector}.
  */
-export abstract class TrackHandler {
+export abstract class TimelineTrackHandler {
     director!: PlayableDirector;
     track!: Models.TrackModel;
 
@@ -136,7 +137,7 @@ class AnimationClipOffsetData {
 }
 
 // TODO: add support for clip clamp modes (loop, pingpong, clamp)
-export class AnimationTrackHandler extends TrackHandler {
+export class TimelineAnimationTrack extends TimelineTrackHandler {
     /** @internal */
     models: Array<Models.ClipModel> = [];
     /** @internal */
@@ -178,7 +179,7 @@ export class AnimationTrackHandler extends TrackHandler {
     onStateChanged() {
         if (this._animator) {
             // We can not check the *isPlaying* state here because the timeline might be paused and evaluated by e.g. ScrollFollow
-            setObjectAnimated(this._animator.gameObject, this, this.director.enabled && this.director.weight > 0);
+            AnimationUtils.setObjectAnimated(this._animator.gameObject, this, this.director.enabled && this.director.weight > 0);
         }
     }
 
@@ -262,7 +263,7 @@ export class AnimationTrackHandler extends TrackHandler {
             // which overrides the timeline
             this._animator = GameObject.getComponent(this.target, Animator) ?? null;
             if (this._animator) {
-                setObjectAnimated(this._animator.gameObject, this, true);
+                AnimationUtils.setObjectAnimated(this._animator.gameObject, this, true);
             }
         }
 
@@ -591,7 +592,21 @@ const muteAudioTracks = getParam("mutetimeline");
 
 declare type AudioClipModel = Models.ClipModel & { _didTriggerPlay: boolean };
 
-export class AudioTrackHandler extends TrackHandler {
+/**
+ * Handles audio playback for a timeline audio track.
+ *
+ * **Runtime mutation:** The track model is read fresh every frame during `evaluate()`.
+ * You can mutate `track.volume`, `clip.start`, `clip.end`, `clip.asset.volume` etc.
+ * at any time — changes take effect on the next frame without rebuilding the timeline.
+ *
+ * **Audio stopping:** Audio clips are automatically stopped when:
+ * - Timeline time moves outside a clip's `[start, end]` range (e.g. jumping or normal playback advancing past a clip)
+ * - The track is muted (via `muted = true`)
+ * - The director is stopped (`director.stop()`)
+ * - The director is paused (`director.pause()`)
+ * - The director is disabled or destroyed
+ */
+export class TimelineAudioTrack extends TimelineTrackHandler {
 
     models: Array<AudioClipModel> = [];
     listener!: AudioListener;
@@ -600,6 +615,10 @@ export class AudioTrackHandler extends TrackHandler {
     lastTime: number = 0;
     audioSource?: AudioSource;
 
+    /** Track-level volume multiplier (0–1). Applied on top of per-clip volume each frame. */
+    get volume(): number { return this.track.volume ?? 1; }
+    set volume(val: number) { this.track.volume = val; }
+
     private _audioLoader: AudioLoader | null = null;
 
     private getAudioFilePath(path: string) {
@@ -796,7 +815,7 @@ export class AudioTrackHandler extends TrackHandler {
     private static _audioBuffers: Map<string, Promise<AudioBuffer | null>> = new Map();
 
     public static dispose() {
-        AudioTrackHandler._audioBuffers.clear();
+        TimelineAudioTrack._audioBuffers.clear();
     }
 
     private handleAudioLoading(model: Models.ClipModel, audio: Audio): Promise<AudioBuffer | null> | null {
@@ -806,8 +825,8 @@ export class AudioTrackHandler extends TrackHandler {
         // TODO: maybe we should cache the loaders / buffers here by path
         const path = this.getAudioFilePath(model.asset.clip);
 
-        if (AudioTrackHandler._audioBuffers.get(path)) {
-            const promise = AudioTrackHandler._audioBuffers.get(path)!
+        if (TimelineAudioTrack._audioBuffers.get(path)) {
+            const promise = TimelineAudioTrack._audioBuffers.get(path)!
             promise.then((buffer) => {
                 if (buffer) audio.setBuffer(buffer);
             });
@@ -827,17 +846,17 @@ export class AudioTrackHandler extends TrackHandler {
                     resolve(null);
                 });
         });
-        AudioTrackHandler._audioBuffers.set(path, loadingPromise);
+        TimelineAudioTrack._audioBuffers.set(path, loadingPromise);
         return loadingPromise;
     }
 }
 
-export class MarkerTrackHandler extends TrackHandler {
+export class TimelineMarkerTrack extends TimelineTrackHandler {
     models: Array<Models.MarkerModel & Record<string, any>> = [];
     needsSorting = true;
 
     *foreachMarker<T>(type: string | null = null) {
-        if(this.needsSorting) this.sort();
+        if (this.needsSorting) this.sort();
         for (const model of this.models) {
             if (model && model.type === type) yield model as T;
         }
@@ -858,26 +877,34 @@ export class MarkerTrackHandler extends TrackHandler {
     }
 }
 
-export class SignalTrackHandler extends TrackHandler {
+export class SignalTrackHandler extends TimelineTrackHandler {
     models: Models.SignalMarkerModel[] = [];
     didTrigger: boolean[] = [];
     receivers: Array<SignalReceiver | null> = [];
 
-    // TODO: test when timeline signals are being reset in Unity
-    // onEnable() {
-    //     for (let i = 0; i < this.didTrigger?.length; i++) {
-    //         this.didTrigger[i] = false;
-    //     }
-    // }
+    private _lastTime: number = -1;
 
-    // private _lastTime: number = -1;
+    onEnable() {
+        // Reset state so signals fire fresh on replay
+        this._lastTime = -1;
+        for (let i = 0; i < this.didTrigger.length; i++) {
+            this.didTrigger[i] = false;
+        }
+    }
+
+    onMuteChanged() {
+        // When unmuted, sync _lastTime to avoid stale comparisons
+        if (!this.track.muted) {
+            this._lastTime = -1;
+        }
+    }
 
     evaluate(time: number) {
         if (this.track.muted) return;
 
-        // let lastTime = this._lastTime;
-        // if (lastTime === -1) lastTime = time;
-        // this._lastTime = time;
+        let lastTime = this._lastTime;
+        if (lastTime === -1) lastTime = time;
+        this._lastTime = time;
 
         const estimatedFrameLengthWithPadding = this.director.context.time.deltaTime * 1.5;
 
@@ -899,7 +926,6 @@ export class SignalTrackHandler extends TrackHandler {
                     isActive = true;
                 }
             }
-            // console.log(time, td, isActive);
             if (isActive) {
                 if (!wasTriggered) {
                     if (debug)
@@ -917,16 +943,59 @@ export class SignalTrackHandler extends TrackHandler {
                     }
                 }
             }
-            else {
-                if (!model.emitOnce)
+            else if (!model.emitOnce) {
+                // Only reset didTrigger when time has moved backward past the signal
+                // (e.g. timeline looped or was scrubbed back). Don't reset just because
+                // the deltaTime-based activation window shrank on a fast frame.
+                if (time < model.time && lastTime >= model.time) {
                     this.didTrigger[i] = false;
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Handles activation (visibility) of bound objects for a timeline activation track.
+ *
+ * Each clip on the track defines a time range during which the bound objects should be active (visible).
+ * @see TimelineTrackHandler for details on how tracks and clips work in general, and how to mutate them at runtime.
+ * @see PlayableDirector for how to control timeline playback and time.
+ * @see TimelineBuilder for how to create and configure timelines and tracks in the editor.
+ */
+export class TimelineActivationTrack extends TimelineTrackHandler {
+
+    evaluate(time: number) {
+        if (this.track.muted) return;
+        // NE-3241 / NE-3737: Don't evaluate when stopped (preserves "Leave as is" post-playback).
+        // But DO evaluate if the user explicitly called director.evaluate().
+        if (!this.director.isPlaying && !this.director._isUserEvaluation) return;
+
+        for (let i = 0; i < this.track.outputs.length; i++) {
+            const binding = this.track.outputs[i];
+            if (typeof binding !== "object" || binding === null) continue;
+
+            let isActive = false;
+            if (this.track.clips) {
+                for (const clip of this.track.clips) {
+                    if (clip.start <= time && time <= clip.end) {
+                        isActive = true;
+                        break;
+                    }
+                }
+            }
+
+            const obj = binding as Object3D;
+            if (obj.visible !== undefined && obj.visible !== isActive) {
+                obj.visible = isActive;
+                if (debug) console.warn(this.director.name, "ActivationTrack-" + i, obj.name, isActive, time);
             }
         }
     }
 }
 
 
-export class ControlTrackHandler extends TrackHandler {
+export class TimelineControlTrack extends TimelineTrackHandler {
     models: Array<Models.ClipModel> = [];
     timelines: Array<PlayableDirector | null> = [];
 

package/src/engine-components/timeline/index.ts

@@ -1,4 +1,5 @@
-export { type ITimelineAnimationCallbacks as ITimelineAnimationOverride } from "./PlayableDirector.js"
+export { DirectorWrapMode, type ITimelineAnimationCallbacks as ITimelineAnimationOverride } from "./PlayableDirector.js"
 export * from "./SignalAsset.js"
+export * from "./TimelineBuilder.js"
 export * from "./TimelineModels.js"
 export * from "./TimelineTracks.js"

package/src/engine-components/ui/Canvas.ts

@@ -4,7 +4,7 @@ import * as ThreeMeshUI from 'three-mesh-ui'
 import { Mathf } from "../../engine/engine_math.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import { FrameEvent } from "../../engine/engine_setup.js";
-import { delayForFrames, getParam } from "../../engine/engine_utils.js";
+import { getParam } from "../../engine/engine_utils.js";
 import { type NeedleXREventArgs } from "../../engine/xr/api.js";
 import { Camera } from "../Camera.js";
 import { GameObject } from "../Component.js";
@@ -236,19 +236,13 @@ export class Canvas extends UIRootComponent implements ICanvas {
         }
     }
 
-    async onEnterXR(args: NeedleXREventArgs) {
+    onEnterXR(args: NeedleXREventArgs) {
         // workaround for https://linear.app/needle/issue/NE-4114
         if (this.screenspace) {
             if (args.xr.isVR || args.xr.isPassThrough) {
                 this.gameObject.visible = false;
             }
         }
-        else {
-            this.gameObject.visible = false;
-            await delayForFrames(1).then(()=>{
-                this.gameObject.visible = true;
-            });
-        }
     }
     onLeaveXR(args: NeedleXREventArgs): void {
         if (this.screenspace) {

package/src/engine-components/ui/Text.ts

@@ -170,12 +170,15 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
             this._font = val;
             this.uiObject?.set(this.getTextOpts());
             this.markDirty();
+            // If the font is assigned during deserialization it means the font URL MUST be resolved with the style suffix. If it's assigned at runtime by a user invocation and is absolute then we assume the user is prividing a full path to the font asset json or png
+            this._assignedAtRuntime = this.__didAwake;
         }
     }
     get font(): string | null {
         return this._font;
     }
     private _font: string | null = "https://cdn.needle.tools/static/fonts/msdf/arial/arial";
+    private _assignedAtRuntime: boolean = true;
 
 
     /**
@@ -274,7 +277,7 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
     }
 
     protected onCreate(_opts: any): void {
-        if (debug) console.log(this);
+        if (debug) console.log("[Text] onCreate", this);
 
         // @marwie : Should be fixed. Currently _opts are always fed with :
         //          backgroundOpacity : color.opacity
@@ -442,7 +445,7 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
     private feedText(text: string, richText: boolean): void {
         // if (!text || text.length <= 0) return;
         // if (!text ) return;
-        if (debug) console.log("feedText", this.uiObject, text, richText);
+        if (debug) console.log("[Text] feedText", this.uiObject, text, richText);
 
         if (!this.uiObject) return;
         if (!this._textMeshUi)
@@ -603,16 +606,17 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
      * @private
      */
     private setFont(opts: ThreeMeshUIEveryOptions, fontStyle: FontStyle) {
-        
+
         if (!this.font) {
-            if(debug) console.warn("No font set for Text component, skipping font setup");
+            if (debug) console.warn("[Text] No font set for Text component, skipping font setup");
             return;
         }
 
         const fontName = this.font;
+
         const familyName = this.getFamilyNameWithCorrectSuffix(fontName, fontStyle);
 
-        if (debug) console.log("Selected font family:" + familyName, this.font);
+        if (debug) console.log(`[Text] Selected font family '${familyName}' from ${fontName} with style ${FontStyle[fontStyle]}`);
 
         // ensure a font family is register under this name
         let fontFamily = ThreeMeshUI.FontLibrary.getFontFamily(familyName as string);
@@ -676,7 +680,7 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
         // e.g. -Medium, -Black, -Thin...
         const styleName = familyName.substring(styleSeparator + 1)?.toLowerCase();
         if (unsupportedStyleNames.includes(styleName)) {
-            if (debug) console.warn("Unsupported font style: " + styleName);
+            if (debug) console.warn("[Text] Unsupported font style: " + styleName);
             return familyName;
         }
 
@@ -691,12 +695,12 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
         const isUpperCase = fontBaseName[0] === fontBaseName[0].toUpperCase();
         const fontNameWithoutSuffix = familyName.substring(0, styleSeparator > pathSeparatorIndex ? styleSeparator : familyName.length);
 
-        if (debug) console.log("Select font: ", familyName, FontStyle[style], fontBaseName, isUpperCase, fontNameWithoutSuffix);
+        if (debug) console.log("[Text] Select font: ", familyName, FontStyle[style], fontBaseName, isUpperCase, fontNameWithoutSuffix);
 
         /**
          * If a user provides a font with an absolute URL AND the font name does not end with "-msdf.json" or ".png" (e.g. "https://example.com/fonts/Arial-Bold"), we will assume that the user is providing the full path to the font files and we will not try to modify the font name based on the style. This allows users to have more control over the font files they are using, especially if they are hosting their own fonts or using a custom font provider that does not follow the same naming conventions as our default fonts.
          */
-        if (isAbsolute && !(familyName.endsWith("-msdf.json") || familyName.endsWith(".png"))) {
+        if (isAbsolute && this._assignedAtRuntime && !(familyName.endsWith("-msdf.json") || familyName.endsWith(".png"))) {
             return fontNameWithoutSuffix;
         }
 

package/src/engine-components/web/CursorFollow.ts

@@ -178,7 +178,6 @@ export class CursorFollow extends Behaviour {
      * - Cursor that follows terrain or mesh surfaces
      *
      * **Important notes:**
-     * - Requires objects in the scene to have colliders for raycasting to work
      * - Works best with {@link keepDistance} set to `false` to allow depth changes
      * - Can be combined with {@link damping} for smooth surface following
      * - The raycast uses the physics system's raycast functionality
@@ -289,20 +288,28 @@ export class CursorFollow extends Behaviour {
 
 
         if (this.snapToSurface) {
-            ray.origin = _position;
-            ray.direction = rayDirection.multiplyScalar(-1);
-            const hits = this.context.physics.raycastFromRay(ray);
-            if (hits?.length) {
-                const hit = hits[0];
-                if (this.damping > 0) {
-                    this.gameObject.worldPosition = _position.lerp(hit.point, this.context.time.deltaTime / this.damping);
-                }
-                else {
-                    this.gameObject.worldPosition = hit.point;
+            ray.origin = cameraPosition;
+            ray.direction = rayDirection;
+            const hits = this.context.physics.raycastFromRay(ray, {
+                testObject: obj => {
+                    return obj !== this.gameObject && !this.gameObject.contains(obj) ? true : false
                 }
-
-                if(debug) {
-                    Gizmos.DrawLine(hit.point, hit.normal!.add(hit.point), 0x00FF00);
+            });
+            if (hits?.length) {
+                // Get the first hit that is not a child of *this* object. Because we do not want to lerp to a surface that is part of the object itself, since that would result in *this* object moving closer and closer to the camera as it tries to snap to itself
+                const hit = hits[0];//.find(h => !this.gameObject.contains(h.object));
+                if (hit) {
+                    if (this.damping > 0) {
+                        this.gameObject.worldPosition = _position.lerp(hit.point, this.context.time.deltaTime / this.damping);
+                        // this._distance = this.gameObject.worldPosition.distanceTo(cameraPosition);
+                    }
+                    else {
+                        this.gameObject.worldPosition = hit.point;
+                        // this._distance = hit.point.distanceTo(cameraPosition);
+                    }
+                    if (debug) {
+                        Gizmos.DrawLine(hit.point, hit.normal!.add(hit.point), 0x00FF00);
+                    }
                 }
             }
         }

package/src/engine-components/webxr/WebXRImageTracking.ts

@@ -886,9 +886,11 @@ async function loadImage(url: string) {
     }
     const promise = new Promise<boolean>(res => {
         _imageElements.set(url, null);
+        if(isDevEnvironment() || debug) console.debug(`[WebXRImageTracking] Start loading image for tracking: ${url}`);
         const imageElement = document.createElement("img") as HTMLImageElement;
         imageElement.src = url;
         imageElement.addEventListener("load", async () => {
+            if(isDevEnvironment() || debug) console.debug(`[WebXRImageTracking] Loaded image for tracking: ${url}`);
             const img = await createImageBitmap(imageElement);
             _imageElements.set(url, img);
             res(true);