@needle-tools/engine 2.63.3-pre → 2.64.0-pre

package/src/engine-components/timeline/PlayableDirector.ts

@@ -84,9 +84,13 @@ export class PlayableDirector extends Behaviour {
     set duration(value: number) { this._duration = value; }
     get weight(): number { return this._weight; };
     set weight(value: number) { this._weight = value; }
+    get speed(): number { return this._speed; }
+    set speed(value: number) { this._speed = value; }
+
 
     private _visibilityChangeEvt?: any;
     private _clonedPlayableAsset: boolean = false;
+    private _speed: number = 1;
 
     awake(): void {
         if (debug)
@@ -149,7 +153,9 @@ export class PlayableDirector extends Behaviour {
 
     play() {
         if (!this.isValid()) return;
+        const pauseChanged = this._isPaused == true;
         this._isPaused = false;
+        if (pauseChanged) this.invokePauseChangedMethodsOnTracks();
         if (this._isPlaying) return;
         this._isPlaying = true;
         this._internalUpdateRoutine = this.startCoroutine(this.internalUpdate());
@@ -161,18 +167,23 @@ export class PlayableDirector extends Behaviour {
         if (this._isPaused) return;
         this._isPaused = true;
         this.evaluate();
+        this.invokePauseChangedMethodsOnTracks();
     }
 
     stop() {
-        for(const track of this._audioTracks) track.stop();
+        for (const track of this._audioTracks) track.stop();
+        const pauseChanged = this._isPaused == true;
+        const wasPlaying = this._isPlaying;
         if (this._isPlaying) {
             this._time = 0;
             this._isPlaying = false;
             this._isPaused = false;
             this.evaluate();
+            if (pauseChanged) this.invokePauseChangedMethodsOnTracks();
         }
         this._isPlaying = false;
         this._isPaused = false;
+        if (pauseChanged && !wasPlaying) this.invokePauseChangedMethodsOnTracks();
         if (this._internalUpdateRoutine)
             this.stopCoroutine(this._internalUpdateRoutine);
         this._internalUpdateRoutine = null;
@@ -183,10 +194,15 @@ export class PlayableDirector extends Behaviour {
         let t = this._time;
         switch (this.extrapolationMode) {
             case DirectorWrapMode.Hold:
-                t = Math.min(t, this._duration);
+                if(this._speed > 0)
+                    t = Math.min(t, this._duration);
+                else if(this._speed < 0)
+                    t = Math.max(t, 0);
+                this._time = t;
                 break;
             case DirectorWrapMode.Loop:
                 t %= this._duration;
+                this._time = t;
                 break;
             case DirectorWrapMode.None:
                 if (t > this._duration) {
@@ -240,10 +256,17 @@ export class PlayableDirector extends Behaviour {
         this._customTracks
     ];
 
+    /** should be called after evaluate if the director was playing */
+    private invokePauseChangedMethodsOnTracks() {
+        for (const track of this.forEachTrack()) {
+            track.onPauseChanged?.call(track);
+        }
+    }
+
     private *internalUpdate() {
         while (this._isPlaying && this.activeAndEnabled) {
             if (!this._isPaused && this._isPlaying) {
-                this._time += this.context.time.deltaTime;
+                this._time += this.context.time.deltaTime * this.speed;
                 this.evaluate();
             }
             // for (let i = 0; i < 5; i++)
@@ -278,10 +301,8 @@ export class PlayableDirector extends Behaviour {
         for (const handler of this._animationTracks) {
             handler.evaluate(time);
         }
-        if (AudioSource.userInteractionRegistered) {
-            for (const handler of this._audioTracks) {
-                handler.evaluate(time);
-            }
+        for (const handler of this._audioTracks) {
+            handler.evaluate(time);
         }
         for (const sig of this._signalTracks) {
             sig.evaluate(time);
@@ -369,8 +390,7 @@ export class PlayableDirector extends Behaviour {
         this._signalTracks.length = 0;
 
         if (!this.playableAsset) return;
-        const audioTracks: Array<Models.TrackModel> = [];
-        const audioAllowedCallbacks : any = [];
+        const audioListener: AudioListener | null = GameObject.findObjectOfType(AudioListener, this.context);
         for (const track of this.playableAsset!.tracks) {
             const type = track.type;
             const registered = PlayableDirector.createTrackFunctions[type];
@@ -442,16 +462,12 @@ export class PlayableDirector extends Behaviour {
                 audio.director = this;
                 audio.track = track;
                 this._audioTracks.push(audio);
-
-                audioAllowedCallbacks.push(() => {
-                    const listener = GameObject.findObjectOfType(AudioListener, this.context) as AudioListener;
-                    if (!listener) return;
-                    audio.listener = listener.listener;
-                    for (let i = 0; i < track.clips.length; i++) {
-                        const clipModel = track.clips[i];
-                        audio.addModel(clipModel);
-                    }
-                });
+                if (!audioListener) continue;
+                audio.listener = audioListener.listener;
+                for (let i = 0; i < track.clips.length; i++) {
+                    const clipModel = track.clips[i];
+                    audio.addModel(clipModel);
+                }
             }
             else if (track.type === Models.TrackType.Marker) {
                 const signalHandler: Tracks.SignalTrackHandler = new Tracks.SignalTrackHandler();
@@ -497,11 +513,6 @@ export class PlayableDirector extends Behaviour {
                 this._controlTracks.push(handler);
             }
         }
-
-        AudioSource.registerWaitForAllowAudio(() => {
-            audioAllowedCallbacks.forEach((cb: any) => cb());
-            audioAllowedCallbacks.length = 0;
-        });
     }
 
     private setAudioTracksAllowPlaying(allow: boolean) {

package/src/engine-components/timeline/TimelineTracks.ts

@@ -6,6 +6,8 @@ import { Context } from "../../engine/engine_setup";
 import { SignalReceiver } from "./SignalAsset";
 import { AnimationClip, Quaternion, Vector3 } from "three";
 import { getParam, getPath } from "../../engine/engine_utils";
+import { AudioSource } from "../AudioSource";
+import { Animator } from "../Animator"
 
 const debug = getParam("debugtimeline");
 
@@ -40,6 +42,7 @@ export abstract class TrackHandler {
     onDestroy?();
     abstract evaluate(time: number);
     onMuteChanged?();
+    onPauseChanged?();
 
     getClipTime(time: number, model: Models.ClipModel) {
         return model.clipIn + (time - model.start) * model.timeScale;
@@ -136,6 +139,16 @@ export class AnimationTrackHandler extends TrackHandler {
     /** holds data/info about clips differences */
     private _actionOffsets: Array<AnimationClipOffsetData> = [];
     private _didBind: boolean = false;
+    private _animator: Animator | null = null;
+    private _animatorWasEnabled?: boolean = false;
+
+    onPauseChanged() {
+        // When the timeline is paused the original animator will be enabled again if it was before
+        if (this._animator && this._animatorWasEnabled !== undefined) {
+            this._animator.enabled = this.director.isPaused ? this._animatorWasEnabled : false;
+        }
+    }
+
 
     createHooks(clipModel: Models.AnimationClipModel, clip) {
         if (clip.tracks?.length <= 0) {
@@ -208,6 +221,13 @@ export class AnimationTrackHandler extends TrackHandler {
             this._actionOffsets.push(off);
         }
 
+        if (this.target) {
+            // We need to disable the animator component in case it also animates
+            // which overrides the timeline
+            this._animator = GameObject.getComponent(this.target, Animator) ?? null;
+            this._animatorWasEnabled = this._animator?.enabled;
+        }
+
         // Clip Offsets
         for (const model of this.models) {
             const clipData = model.asset as Models.AnimationClipModel;
@@ -261,6 +281,8 @@ export class AnimationTrackHandler extends TrackHandler {
         if (!this.mixer) return;
         this.bind();
 
+        if (this._animator && this.director.isPlaying && this.director.weight > 0) this._animator.enabled = false;
+
         this._totalOffsetPosition.set(0, 0, 0);
         this._totalOffsetRotation.set(0, 0, 0, 1);
         this._totalOffsetPosition2.set(0, 0, 0);
@@ -268,38 +290,60 @@ export class AnimationTrackHandler extends TrackHandler {
         let activeClips = 0;
         let blend: number = 0;
         let didPostExtrapolate = false;
+        let didPreExtrapolate = false;
         for (let i = 0; i < this.clips.length; i++) {
             const model = this.models[i];
             const action = this.actions[i];
             const clipModel = model.asset as Models.AnimationClipModel;
             action.weight = 0;
+
             const isInTimeRange = time >= model.start && time <= model.end;
+            const preExtrapolation: Models.ClipExtrapolation = model.preExtrapolationMode;
             const postExtrapolation: Models.ClipExtrapolation = model.postExtrapolationMode;
+            const nextClip = i < this.clips.length - 1 ? this.models[i + 1] : null;
             let isActive = isInTimeRange;
+            let doPreExtrapolate = false;
+
             if (!isActive && !didPostExtrapolate && model.end < time && postExtrapolation !== Models.ClipExtrapolation.None) {
-                const nextClip = i < this.clips.length - 1 ? this.models[i + 1] : null;
                 // use post extrapolate if its the last clip of the next clip has not yet started
                 if (!nextClip || nextClip.start > time) {
                     isActive = true;
                     didPostExtrapolate = true;
                 }
             }
-            const preExtrapolation: Models.ClipExtrapolation = model.preExtrapolationMode;
-            if (i == 0 && !isActive && !didPostExtrapolate && model.start > time && preExtrapolation !== Models.ClipExtrapolation.None) {
-                isActive = true;
-                if (preExtrapolation !== Models.ClipExtrapolation.Hold)
-                    time += model.start;
+            else if (i == 0 && !isActive && !didPreExtrapolate && model.start > time && preExtrapolation !== Models.ClipExtrapolation.None) {
+                if (!nextClip || nextClip.start < time) {
+                    isActive = true;
+                    doPreExtrapolate = true;
+                    didPreExtrapolate = true;
+                }
             }
+
             if (isActive) {
                 // const clip = this.clips[i];
                 let weight = 1;
                 weight *= this.evaluateWeight(time, i, this.models, isActive);
+
+                let handleLoop = isInTimeRange;
+                if(doPreExtrapolate){
+                    if (preExtrapolation !== Models.ClipExtrapolation.Hold) {
+                        time += model.start;
+                        handleLoop = true;
+                    }
+                }
+
                 // TODO: handle clipIn again
                 let t = this.getClipTime(time, model);
                 let loops = 0;
                 const duration = clipModel.duration;
 
-                if (isInTimeRange) {
+                if (doPreExtrapolate) {
+                    if (preExtrapolation === Models.ClipExtrapolation.Hold) {
+                        t = 0;
+                    }
+                }
+
+                if (handleLoop) {
                     if (clipModel.loop) {
                         // const t0 = t - .001;
                         loops += Math.floor(t / (duration + .000001));
@@ -468,6 +512,8 @@ export class AudioTrackHandler extends TrackHandler {
     audioContextTimeOffset: Array<number> = [];
     lastTime: number = 0;
 
+    private _audioLoader: THREE.AudioLoader | null = null;
+
     private getAudioFilePath(path: string) {
         // TODO: this should be the timeline asset location probably which MIGHT be different
         const glbLocation = this.director.sourceId;
@@ -483,18 +529,9 @@ export class AudioTrackHandler extends TrackHandler {
     }
 
     addModel(model: Models.ClipModel) {
-        const path = this.getAudioFilePath(model.asset.clip);
         const audio = new THREE.Audio(this.listener);
-        audio.setVolume(model.asset.volume);
-        const loader = new THREE.AudioLoader();
-        if (debug)
-            console.log(path, this.director.sourceId);
-        loader.load(path, (buffer) => {
-            audio.setBuffer(buffer);
-            audio.loop = model.asset.loop;
-            this.audio.push(audio);
-            this.models.push(model);
-        });
+        this.audio.push(audio);
+        this.models.push(model);
     }
 
     onDisable() {
@@ -521,13 +558,31 @@ export class AudioTrackHandler extends TrackHandler {
                 audio.stop();
         }
     }
+
     evaluate(time: number) {
         if (muteAudioTracks) return;
         if (this.track.muted) return;
+        if (this.director.speed < 0) {
+            // Reversed audio playback is currently not supported
+            return;
+        }
+        const isMuted = this.director.context.application.muted;
+        // this is just so that we dont hear the very first beat when the audio starts but is muted
+        // if we dont add a delay we hear a little bit of the audio before it shuts down
+        // MAYBE instead of doing it like this we should connect a custom audio node (or disconnect the output node?)
+        const playTimeOffset = isMuted ? .1 : 0;
         for (let i = 0; i < this.models.length; i++) {
             const model = this.models[i];
             const audio = this.audio[i];
-            if (time >= model.start && time <= model.end) {
+            // only trigger loading for tracks that are CLOSE to being played
+            if ((!audio || !audio.buffer) && this.isInTimeRange(model, time - 1, time + 1)) {
+                this.handleAudioLoading(model, audio);
+            }
+            if (AudioSource.userInteractionRegistered === false) continue;
+            if (audio === null || !audio.buffer) continue;
+            audio.playbackRate = this.director.context.time.timeScale;
+            audio.loop = model.asset.loop;
+            if (time >= model.start && time <= model.end && time < this.director.duration) {
                 if (this.director.isPlaying == false) {
                     if (audio.isPlaying)
                         audio.stop();
@@ -535,7 +590,7 @@ export class AudioTrackHandler extends TrackHandler {
                 }
                 else if (!audio.isPlaying) {
                     audio.offset = model.clipIn + (time - model.start) * model.timeScale;
-                    audio.play();
+                    audio.play(playTimeOffset);
                 }
                 else {
                     const targetOffset = model.clipIn + (time - model.start) * model.timeScale;
@@ -547,10 +602,11 @@ export class AudioTrackHandler extends TrackHandler {
                     if (diff > 0.3) {
                         audio.offset = targetOffset;
                         audio.stop();
-                        audio.play();
+                        audio.play(playTimeOffset);
                     }
                 }
-                let vol = model.asset.volume;
+                let vol = model.asset.volume as number;
+                if (isMuted) vol = 0;
                 if (model.easeInDuration > 0) {
                     const easeIn = Math.min((time - model.start) / model.easeInDuration, 1);
                     vol *= easeIn;
@@ -568,6 +624,60 @@ export class AudioTrackHandler extends TrackHandler {
         }
         this.lastTime = time;
     }
+
+    /** Call to load audio buffer for a specific time in the track. Can be used to preload the timeline audio */
+    loadAudio(time: number, lookAhead: number = 0, lookBehind: number = 0) {
+        let promises: Array<Promise<AudioBuffer | null>> | null = null;
+        const rangeStart = time - lookBehind;
+        const rangeEnd = time + lookAhead;
+        for (const model of this.models) {
+            if (this.isInTimeRange(model, rangeStart, rangeEnd)) {
+                const audio = this.audio[this.models.indexOf(model)];
+                const promise = this.handleAudioLoading(model, audio);
+                if (promise !== null) {
+                    if (promises === null) promises = [];
+                    promises.push(promise);
+                }
+            }
+        }
+        if (promises !== null) {
+            return Promise.all(promises);
+        }
+        return null;
+    }
+
+    private isInTimeRange(model: Models.ClipModel, start: number, end: number) {
+        // Range surrounds clip range
+        if (start <= model.start && end >= model.end) return true;
+        // Range start is in clip range
+        if (start >= model.start && start <= model.end) return true;
+        // Range end is in clip range
+        if (end >= model.start && end <= model.end) return true;
+        return false;
+    }
+
+    private handleAudioLoading(model: Models.ClipModel, audio: THREE.Audio): Promise<AudioBuffer | null> | null {
+        if (!this._audioLoader) {
+            const path = this.getAudioFilePath(model.asset.clip);
+            this._audioLoader = new THREE.AudioLoader();
+            // TODO: maybe we should cache the loaders / buffers here by path
+            if (debug) console.warn("LOAD audio track", path, this.director.sourceId);
+            const loadingPromise = new Promise<AudioBuffer | null>((resolve, _reject) => {
+                this._audioLoader!.load(path,
+                    buffer => {
+                        audio.setBuffer(buffer);
+                        resolve(buffer);
+                    },
+                    undefined,
+                    err => {
+                        console.error("Error loading audio", err);
+                        resolve(null);
+                    });
+            });
+            return loadingPromise;
+        }
+        return null;
+    }
 }
 
 export class SignalTrackHandler extends TrackHandler {

package/src/tsconfig.json

@@ -0,0 +1,33 @@
+{
+  "compilerOptions": {   
+    "target": "ESNext", 
+    "module": "ESNext",
+    "outDir": "dist",
+    "useDefineForClassFields": true,
+    "lib": ["ESNext", "DOM"],
+    "moduleResolution": "Node",
+    "strict": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "esModuleInterop": true,
+    "noEmit": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "noImplicitAny": false,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "experimentalDecorators": true,
+    "skipLibCheck": true,
+    "allowJs": true,
+    "types": [
+      "three",
+      "vite/client"
+    ]
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": ["node_modules", "dist"]
+}

package/src/engine/codegen/license.js

@@ -1 +0,0 @@
-export const hasLicense = false;