@needle-tools/engine 2.63.3-pre.1 → 2.64.0-pre

package/src/engine-components/Renderer.ts

@@ -5,7 +5,7 @@ import { RendererLightmap } from "./RendererLightmap";
 import { Context, FrameEvent } from "../engine/engine_setup";
 import { getParam } from "../engine/engine_utils";
 import { serializable } from "../engine/engine_serialization_decorator";
-import { AxesHelper, Material, Mesh, Object3D, SkinnedMesh, Texture, Vector4 } from "three";
+import { AxesHelper, Material, Matrix4, Mesh, Object3D, SkinnedMesh, Texture, Vector4 } from "three";
 import { NEEDLE_render_objects } from "../engine/extensions/NEEDLE_render_objects";
 import { NEEDLE_progressive } from "../engine/extensions/NEEDLE_progressive";
 import { NEED_UPDATE_INSTANCE_KEY } from "../engine/engine_instancing";
@@ -18,6 +18,7 @@ import { showBalloonWarning } from "../engine/debug/debug";
 // for staying compatible with old code
 export { InstancingUtil } from "../engine/engine_instancing";
 
+const debugRenderer = getParam("debugrenderer");
 const suppressInstancing = getParam("noInstancing");
 const debugLightmap = getParam("debuglightmaps") ? true : false;
 const debugInstancing = getParam("debuginstancing");
@@ -294,6 +295,7 @@ export class Renderer extends Behaviour implements IRenderer {
     }
 
     awake() {
+        if (debugRenderer) console.log("Renderer ", this.name, this);
         this.clearInstancingState();
 
         if (this.probeAnchor && debug) this.probeAnchor.add(new AxesHelper(.2));
@@ -332,6 +334,9 @@ export class Renderer extends Behaviour implements IRenderer {
             if (this.renderOrder !== undefined && this.renderOrder.length > 0)
                 this.gameObject.renderOrder = this.renderOrder[0];
         }
+        else {
+            this.context.addBeforeRenderListener(this.gameObject, this.onBeforeRenderThree.bind(this));
+        }
 
         if (this.lightmapIndex >= 0) {
             // use the override lightmap if its not undefined
@@ -394,20 +399,14 @@ export class Renderer extends Behaviour implements IRenderer {
         this.handles = undefined;
         this.prevLayers = undefined;
     }
+
     setInstancingEnabled(enabled: boolean): boolean {
         if (this._isInstancingEnabled === enabled) return enabled && (this.handles === undefined || this.handles != null && this.handles.length > 0);
         this._isInstancingEnabled = enabled;
         if (enabled) {
-            // if handles is undefined we
             if (this.handles === undefined) {
-                this.handles = instancing.setup(this.gameObject, this.context, null, { rend: this, foundMeshes: 0 });
+                this.handles = instancing.setup(this, this.gameObject, this.context, null, { rend: this, foundMeshes: 0, useMatrixWorldAutoUpdate: this.useInstanceMatrixWorldAutoUpdate() });
                 if (this.handles) {
-                    // const disableSelf = this.gameObject.type === "Mesh" || this.gameObject.children?.length === this.handles.length;
-                    // this.gameObject.visible = !disableSelf;
-                    // this.gameObject.type = "Object3D";
-                    // this.gameObject.material = null;
-                    // console.log("Using instancing", this.gameObject.visible);
-                    // this.gameObject.onBeforeRender = () => console.log("SHOULD NOT BE CALLED");
                     GameObject.markAsInstancedRendered(this.gameObject, true);
                     return true;
                 }
@@ -417,8 +416,6 @@ export class Renderer extends Behaviour implements IRenderer {
                     handler.updateInstanceMatrix(true);
                     handler.add();
                 }
-                // this.gameObject.type = "Object3D";
-                // this.gameObject.visible = false;
                 GameObject.markAsInstancedRendered(this.gameObject, true);
                 return true;
             }
@@ -429,13 +426,19 @@ export class Renderer extends Behaviour implements IRenderer {
                     handler.remove();
                 }
             }
-            // this.gameObject.visible = true;
             return true;
         }
 
         return false;
     }
 
+    /** Return true to wrap matrix update events for instanced rendering to update instance matrices automatically when matrixWorld changes
+     * This is a separate method to be overrideable from user code
+     */
+    useInstanceMatrixWorldAutoUpdate() {
+        return true;
+    }
+
     start() {
         if (this.enableInstancing && !suppressInstancing) {
             this.setInstancingEnabled(true);
@@ -461,6 +464,7 @@ export class Renderer extends Behaviour implements IRenderer {
         }
 
         this.updateReflectionProbe();
+
     }
 
     onDisable() {
@@ -479,17 +483,12 @@ export class Renderer extends Behaviour implements IRenderer {
         NEEDLE_render_objects.applyStencil(this);
     }
 
-    static envmap: THREE.Texture | null = null;
 
     onBeforeRender() {
         if (!this.gameObject) {
             return;
         }
 
-        Renderer.envmap = this.scene.environment;
-
-        const needsUpdate: boolean = this.gameObject[NEED_UPDATE_INSTANCE_KEY] === true || this.gameObject.matrixWorldNeedsUpdate;
-
         if (this.isMultiMaterialObject(this.gameObject) && this.gameObject.children?.length > 0) {
             for (const ch of this.gameObject.children) {
                 this.applySettings(ch);
@@ -499,9 +498,14 @@ export class Renderer extends Behaviour implements IRenderer {
             this.applySettings(this.gameObject);
         }
 
-        if (needsUpdate) {
-            delete this.gameObject[NEED_UPDATE_INSTANCE_KEY];
-            if (this.handles) {
+        if (this.handles?.length) {
+            // if (this.name === "Darbouka")
+            //     console.log(this.name, this.gameObject.matrixWorldNeedsUpdate);
+            const needsUpdate: boolean = this.gameObject[NEED_UPDATE_INSTANCE_KEY] === true;// || this.gameObject.matrixWorldNeedsUpdate;
+            if (needsUpdate) {
+                if(debugInstancing)
+                    console.log("UPDATE INSTANCED MATRICES", this.context.time.frame);
+                this.gameObject[NEED_UPDATE_INSTANCE_KEY] = false;
                 const remove = false;// Math.random() < .01;
                 for (let i = this.handles.length - 1; i >= 0; i--) {
                     const h = this.handles[i];
@@ -539,14 +543,7 @@ export class Renderer extends Behaviour implements IRenderer {
 
     onBeforeRenderThree(_renderer, _scene, _camera, _geometry, material, _group) {
 
-        // progressive load before rendering so we only load textures for visible materials
-        if (!suppressProgressiveLoading && material._didRequestTextureLOD === undefined && this.allowProgressiveLoading) {
-            material._didRequestTextureLOD = 0;
-            if (debugProgressiveLoading) {
-                console.log("Load material LOD", material.name);
-            }
-            NEEDLE_progressive.assignTextureLOD(this.context, this.sourceId, material);
-        }
+        this.loadProgressiveTextures(material);
 
         if (material.envMapIntensity !== undefined) {
             const factor = this.hasLightmap ? Math.PI : 1;
@@ -607,6 +604,22 @@ export class Renderer extends Behaviour implements IRenderer {
         }
     }
 
+    loadProgressiveTextures(material: THREE.Material) {
+        // progressive load before rendering so we only load textures for visible materials
+        if (!suppressProgressiveLoading && material) {
+            if (debugProgressiveLoading && material["_didRequestTextureLOD"] === undefined)
+                console.warn("Progressive?", this)
+
+            if (material["_didRequestTextureLOD"] === undefined && this.allowProgressiveLoading) {
+                material["_didRequestTextureLOD"] = 0;
+                if (debugProgressiveLoading) {
+                    console.log("Load material LOD", material.name);
+                }
+                NEEDLE_progressive.assignTextureLOD(this.context, this.sourceId, material);
+            }
+        }
+    }
+
     private applySettings(go: THREE.Object3D) {
         go.receiveShadow = this.receiveShadows;
         if (this.shadowCastingMode == ShadowCastingMode.On) {
@@ -689,17 +702,22 @@ export enum ShadowCastingMode {
 
 
 
-declare class InstancingSetupArgs { rend: Renderer; foundMeshes: number };
+declare class InstancingSetupArgs {
+    rend: Renderer;
+    foundMeshes: number;
+    useMatrixWorldAutoUpdate: boolean;
+};
 
 class InstancingHandler {
 
     public objs: InstancedMeshRenderer[] = [];
 
-    public setup(obj: THREE.Object3D, context: Context, handlesArray: InstanceHandle[] | null, args: InstancingSetupArgs, level: number = 0)
+    public setup(renderer:Renderer, obj: THREE.Object3D, context: Context, handlesArray: InstanceHandle[] | null, args: InstancingSetupArgs, level: number = 0)
         : InstanceHandle[] | null {
 
         const res = this.tryCreateOrAddInstance(obj, context, args);
         if (res) {
+            renderer.loadProgressiveTextures(res.instancer.material);
             if (handlesArray === null) handlesArray = [];
             handlesArray.push(res);
             return handlesArray;
@@ -708,7 +726,7 @@ class InstancingHandler {
         if (level <= 0 && obj.type !== "Mesh") {
             const nextLevel = level + 1;
             for (const ch of obj.children) {
-                handlesArray = this.setup(ch, context, handlesArray, args, nextLevel);
+                handlesArray = this.setup(renderer, ch, context, handlesArray, args, nextLevel);
             }
         }
         return handlesArray;
@@ -736,16 +754,52 @@ class InstancingHandler {
             for (const i of this.objs) {
                 if (i.isFull()) continue;
                 if (i.geo === geo && i.material === mat) {
-                    return i.addInstance(mesh);
+                    const handle = i.addInstance(mesh);
+                    if (args.useMatrixWorldAutoUpdate && handle)
+                        this.autoUpdateInstanceMatrix(mesh, i, handle);
+                    return handle;
                 }
             }
             // console.log("Add new instance mesh renderer", obj);
             const i = new InstancedMeshRenderer(obj.name, geo, mat, 200, context);
             this.objs.push(i);
-            return i.addInstance(mesh);
+            const handle = i.addInstance(mesh);
+            if (args.useMatrixWorldAutoUpdate && handle)
+                this.autoUpdateInstanceMatrix(mesh, i, handle);
+            return handle;
         }
         return null;
     }
+
+    private autoUpdateInstanceMatrix(obj: Object3D, _renderer: InstancedMeshRenderer, _handle: InstanceHandle) {
+        const original = obj.matrixWorld["multiplyMatrices"].bind(obj.matrixWorld);
+        let previousMatrix: THREE.Matrix4 = obj.matrixWorld.clone();
+
+        const matrixChangeWrapper = (a: Matrix4, b: Matrix4) => {
+            const newMatrixWorld = original(a, b);
+            // console.warn("MULT", obj.matrixWorldNeedsUpdate);
+            if (obj[NEED_UPDATE_INSTANCE_KEY] || previousMatrix.equals(newMatrixWorld) === false) {
+                previousMatrix.copy(newMatrixWorld)
+                // handle.setMatrix(newMatrixWorld);
+                obj[NEED_UPDATE_INSTANCE_KEY] = true;
+            }
+            return newMatrixWorld;
+        };
+        obj.matrixWorld["multiplyMatrices"] = matrixChangeWrapper;
+
+        // wrap matrixWorldNeedsUpdate
+        // let originalMatrixWorldNeedsUpdate = obj.matrixWorldNeedsUpdate;
+        // Object.defineProperty(obj, "matrixWorldNeedsUpdate", {
+        //     get: () => {
+        //         return originalMatrixWorldNeedsUpdate;
+        //     },
+        //     set: (value: boolean) => {
+        //         if(value) console.warn("SET MATRIX WORLD NEEDS UPDATE");
+        //         originalMatrixWorldNeedsUpdate = value;
+        //     }
+        // });
+
+    }
 }
 const instancing: InstancingHandler = new InstancingHandler();
 
@@ -773,6 +827,11 @@ class InstanceHandle {
         this.instancer.updateInstance(this.object.matrixWorld, this.instanceIndex);
     }
 
+    setMatrix(matrix: THREE.Matrix4) {
+        if (this.instanceIndex < 0) return;
+        this.instancer.updateInstance(matrix, this.instanceIndex);
+    }
+
     add() {
         if (this.instanceIndex >= 0) return;
         this.instancer.add(this);

package/src/engine-components/js-extensions/Object3D.ts

@@ -12,7 +12,6 @@ export function apply(object: Object3D) {
     }
 }
 
-
 Object3D.prototype["SetActive"] = function (active: boolean) {
     this.visible = active;
 }
@@ -69,6 +68,13 @@ if (!Object.getOwnPropertyDescriptor(Object3D.prototype, "activeSelf")) {
     });
 }
 
+if (!Object.getOwnPropertyDescriptor(Object3D.prototype, "transform")) {
+    Object.defineProperty(Object3D.prototype, "transform", {
+        get: function () {
+            return this;
+        }
+    });
+}
 
 
 

package/src/engine-components/timeline/PlayableDirector.ts

@@ -84,9 +84,13 @@ export class PlayableDirector extends Behaviour {
     set duration(value: number) { this._duration = value; }
     get weight(): number { return this._weight; };
     set weight(value: number) { this._weight = value; }
+    get speed(): number { return this._speed; }
+    set speed(value: number) { this._speed = value; }
+
 
     private _visibilityChangeEvt?: any;
     private _clonedPlayableAsset: boolean = false;
+    private _speed: number = 1;
 
     awake(): void {
         if (debug)
@@ -149,7 +153,9 @@ export class PlayableDirector extends Behaviour {
 
     play() {
         if (!this.isValid()) return;
+        const pauseChanged = this._isPaused == true;
         this._isPaused = false;
+        if (pauseChanged) this.invokePauseChangedMethodsOnTracks();
         if (this._isPlaying) return;
         this._isPlaying = true;
         this._internalUpdateRoutine = this.startCoroutine(this.internalUpdate());
@@ -161,18 +167,23 @@ export class PlayableDirector extends Behaviour {
         if (this._isPaused) return;
         this._isPaused = true;
         this.evaluate();
+        this.invokePauseChangedMethodsOnTracks();
     }
 
     stop() {
-        for(const track of this._audioTracks) track.stop();
+        for (const track of this._audioTracks) track.stop();
+        const pauseChanged = this._isPaused == true;
+        const wasPlaying = this._isPlaying;
         if (this._isPlaying) {
             this._time = 0;
             this._isPlaying = false;
             this._isPaused = false;
             this.evaluate();
+            if (pauseChanged) this.invokePauseChangedMethodsOnTracks();
         }
         this._isPlaying = false;
         this._isPaused = false;
+        if (pauseChanged && !wasPlaying) this.invokePauseChangedMethodsOnTracks();
         if (this._internalUpdateRoutine)
             this.stopCoroutine(this._internalUpdateRoutine);
         this._internalUpdateRoutine = null;
@@ -183,10 +194,15 @@ export class PlayableDirector extends Behaviour {
         let t = this._time;
         switch (this.extrapolationMode) {
             case DirectorWrapMode.Hold:
-                t = Math.min(t, this._duration);
+                if(this._speed > 0)
+                    t = Math.min(t, this._duration);
+                else if(this._speed < 0)
+                    t = Math.max(t, 0);
+                this._time = t;
                 break;
             case DirectorWrapMode.Loop:
                 t %= this._duration;
+                this._time = t;
                 break;
             case DirectorWrapMode.None:
                 if (t > this._duration) {
@@ -240,10 +256,17 @@ export class PlayableDirector extends Behaviour {
         this._customTracks
     ];
 
+    /** should be called after evaluate if the director was playing */
+    private invokePauseChangedMethodsOnTracks() {
+        for (const track of this.forEachTrack()) {
+            track.onPauseChanged?.call(track);
+        }
+    }
+
     private *internalUpdate() {
         while (this._isPlaying && this.activeAndEnabled) {
             if (!this._isPaused && this._isPlaying) {
-                this._time += this.context.time.deltaTime;
+                this._time += this.context.time.deltaTime * this.speed;
                 this.evaluate();
             }
             // for (let i = 0; i < 5; i++)
@@ -278,10 +301,8 @@ export class PlayableDirector extends Behaviour {
         for (const handler of this._animationTracks) {
             handler.evaluate(time);
         }
-        if (AudioSource.userInteractionRegistered) {
-            for (const handler of this._audioTracks) {
-                handler.evaluate(time);
-            }
+        for (const handler of this._audioTracks) {
+            handler.evaluate(time);
         }
         for (const sig of this._signalTracks) {
             sig.evaluate(time);
@@ -369,8 +390,7 @@ export class PlayableDirector extends Behaviour {
         this._signalTracks.length = 0;
 
         if (!this.playableAsset) return;
-        const audioTracks: Array<Models.TrackModel> = [];
-        const audioAllowedCallbacks : any = [];
+        const audioListener: AudioListener | null = GameObject.findObjectOfType(AudioListener, this.context);
         for (const track of this.playableAsset!.tracks) {
             const type = track.type;
             const registered = PlayableDirector.createTrackFunctions[type];
@@ -442,16 +462,12 @@ export class PlayableDirector extends Behaviour {
                 audio.director = this;
                 audio.track = track;
                 this._audioTracks.push(audio);
-
-                audioAllowedCallbacks.push(() => {
-                    const listener = GameObject.findObjectOfType(AudioListener, this.context) as AudioListener;
-                    if (!listener) return;
-                    audio.listener = listener.listener;
-                    for (let i = 0; i < track.clips.length; i++) {
-                        const clipModel = track.clips[i];
-                        audio.addModel(clipModel);
-                    }
-                });
+                if (!audioListener) continue;
+                audio.listener = audioListener.listener;
+                for (let i = 0; i < track.clips.length; i++) {
+                    const clipModel = track.clips[i];
+                    audio.addModel(clipModel);
+                }
             }
             else if (track.type === Models.TrackType.Marker) {
                 const signalHandler: Tracks.SignalTrackHandler = new Tracks.SignalTrackHandler();
@@ -497,11 +513,6 @@ export class PlayableDirector extends Behaviour {
                 this._controlTracks.push(handler);
             }
         }
-
-        AudioSource.registerWaitForAllowAudio(() => {
-            audioAllowedCallbacks.forEach((cb: any) => cb());
-            audioAllowedCallbacks.length = 0;
-        });
     }
 
     private setAudioTracksAllowPlaying(allow: boolean) {

package/src/engine-components/timeline/TimelineTracks.ts

@@ -6,6 +6,8 @@ import { Context } from "../../engine/engine_setup";
 import { SignalReceiver } from "./SignalAsset";
 import { AnimationClip, Quaternion, Vector3 } from "three";
 import { getParam, getPath } from "../../engine/engine_utils";
+import { AudioSource } from "../AudioSource";
+import { Animator } from "../Animator"
 
 const debug = getParam("debugtimeline");
 
@@ -40,6 +42,7 @@ export abstract class TrackHandler {
     onDestroy?();
     abstract evaluate(time: number);
     onMuteChanged?();
+    onPauseChanged?();
 
     getClipTime(time: number, model: Models.ClipModel) {
         return model.clipIn + (time - model.start) * model.timeScale;
@@ -136,6 +139,16 @@ export class AnimationTrackHandler extends TrackHandler {
     /** holds data/info about clips differences */
     private _actionOffsets: Array<AnimationClipOffsetData> = [];
     private _didBind: boolean = false;
+    private _animator: Animator | null = null;
+    private _animatorWasEnabled?: boolean = false;
+
+    onPauseChanged() {
+        // When the timeline is paused the original animator will be enabled again if it was before
+        if (this._animator && this._animatorWasEnabled !== undefined) {
+            this._animator.enabled = this.director.isPaused ? this._animatorWasEnabled : false;
+        }
+    }
+
 
     createHooks(clipModel: Models.AnimationClipModel, clip) {
         if (clip.tracks?.length <= 0) {
@@ -208,6 +221,13 @@ export class AnimationTrackHandler extends TrackHandler {
             this._actionOffsets.push(off);
         }
 
+        if (this.target) {
+            // We need to disable the animator component in case it also animates
+            // which overrides the timeline
+            this._animator = GameObject.getComponent(this.target, Animator) ?? null;
+            this._animatorWasEnabled = this._animator?.enabled;
+        }
+
         // Clip Offsets
         for (const model of this.models) {
             const clipData = model.asset as Models.AnimationClipModel;
@@ -261,6 +281,8 @@ export class AnimationTrackHandler extends TrackHandler {
         if (!this.mixer) return;
         this.bind();
 
+        if (this._animator && this.director.isPlaying && this.director.weight > 0) this._animator.enabled = false;
+
         this._totalOffsetPosition.set(0, 0, 0);
         this._totalOffsetRotation.set(0, 0, 0, 1);
         this._totalOffsetPosition2.set(0, 0, 0);
@@ -268,38 +290,60 @@ export class AnimationTrackHandler extends TrackHandler {
         let activeClips = 0;
         let blend: number = 0;
         let didPostExtrapolate = false;
+        let didPreExtrapolate = false;
         for (let i = 0; i < this.clips.length; i++) {
             const model = this.models[i];
             const action = this.actions[i];
             const clipModel = model.asset as Models.AnimationClipModel;
             action.weight = 0;
+
             const isInTimeRange = time >= model.start && time <= model.end;
+            const preExtrapolation: Models.ClipExtrapolation = model.preExtrapolationMode;
             const postExtrapolation: Models.ClipExtrapolation = model.postExtrapolationMode;
+            const nextClip = i < this.clips.length - 1 ? this.models[i + 1] : null;
             let isActive = isInTimeRange;
+            let doPreExtrapolate = false;
+
             if (!isActive && !didPostExtrapolate && model.end < time && postExtrapolation !== Models.ClipExtrapolation.None) {
-                const nextClip = i < this.clips.length - 1 ? this.models[i + 1] : null;
                 // use post extrapolate if its the last clip of the next clip has not yet started
                 if (!nextClip || nextClip.start > time) {
                     isActive = true;
                     didPostExtrapolate = true;
                 }
             }
-            const preExtrapolation: Models.ClipExtrapolation = model.preExtrapolationMode;
-            if (i == 0 && !isActive && !didPostExtrapolate && model.start > time && preExtrapolation !== Models.ClipExtrapolation.None) {
-                isActive = true;
-                if (preExtrapolation !== Models.ClipExtrapolation.Hold)
-                    time += model.start;
+            else if (i == 0 && !isActive && !didPreExtrapolate && model.start > time && preExtrapolation !== Models.ClipExtrapolation.None) {
+                if (!nextClip || nextClip.start < time) {
+                    isActive = true;
+                    doPreExtrapolate = true;
+                    didPreExtrapolate = true;
+                }
             }
+
             if (isActive) {
                 // const clip = this.clips[i];
                 let weight = 1;
                 weight *= this.evaluateWeight(time, i, this.models, isActive);
+
+                let handleLoop = isInTimeRange;
+                if(doPreExtrapolate){
+                    if (preExtrapolation !== Models.ClipExtrapolation.Hold) {
+                        time += model.start;
+                        handleLoop = true;
+                    }
+                }
+
                 // TODO: handle clipIn again
                 let t = this.getClipTime(time, model);
                 let loops = 0;
                 const duration = clipModel.duration;
 
-                if (isInTimeRange) {
+                if (doPreExtrapolate) {
+                    if (preExtrapolation === Models.ClipExtrapolation.Hold) {
+                        t = 0;
+                    }
+                }
+
+                if (handleLoop) {
                     if (clipModel.loop) {
                         // const t0 = t - .001;
                         loops += Math.floor(t / (duration + .000001));
@@ -468,6 +512,8 @@ export class AudioTrackHandler extends TrackHandler {
     audioContextTimeOffset: Array<number> = [];
     lastTime: number = 0;
 
+    private _audioLoader: THREE.AudioLoader | null = null;
+
     private getAudioFilePath(path: string) {
         // TODO: this should be the timeline asset location probably which MIGHT be different
         const glbLocation = this.director.sourceId;
@@ -483,18 +529,9 @@ export class AudioTrackHandler extends TrackHandler {
     }
 
     addModel(model: Models.ClipModel) {
-        const path = this.getAudioFilePath(model.asset.clip);
         const audio = new THREE.Audio(this.listener);
-        audio.setVolume(model.asset.volume);
-        const loader = new THREE.AudioLoader();
-        if (debug)
-            console.log(path, this.director.sourceId);
-        loader.load(path, (buffer) => {
-            audio.setBuffer(buffer);
-            audio.loop = model.asset.loop;
-            this.audio.push(audio);
-            this.models.push(model);
-        });
+        this.audio.push(audio);
+        this.models.push(model);
     }
 
     onDisable() {
@@ -521,9 +558,14 @@ export class AudioTrackHandler extends TrackHandler {
                 audio.stop();
         }
     }
+
     evaluate(time: number) {
         if (muteAudioTracks) return;
         if (this.track.muted) return;
+        if (this.director.speed < 0) {
+            // Reversed audio playback is currently not supported
+            return;
+        }
         const isMuted = this.director.context.application.muted;
         // this is just so that we dont hear the very first beat when the audio starts but is muted
         // if we dont add a delay we hear a little bit of the audio before it shuts down
@@ -532,7 +574,15 @@ export class AudioTrackHandler extends TrackHandler {
         for (let i = 0; i < this.models.length; i++) {
             const model = this.models[i];
             const audio = this.audio[i];
-            if (time >= model.start && time <= model.end) {
+            // only trigger loading for tracks that are CLOSE to being played
+            if ((!audio || !audio.buffer) && this.isInTimeRange(model, time - 1, time + 1)) {
+                this.handleAudioLoading(model, audio);
+            }
+            if (AudioSource.userInteractionRegistered === false) continue;
+            if (audio === null || !audio.buffer) continue;
+            audio.playbackRate = this.director.context.time.timeScale;
+            audio.loop = model.asset.loop;
+            if (time >= model.start && time <= model.end && time < this.director.duration) {
                 if (this.director.isPlaying == false) {
                     if (audio.isPlaying)
                         audio.stop();
@@ -556,7 +606,7 @@ export class AudioTrackHandler extends TrackHandler {
                     }
                 }
                 let vol = model.asset.volume as number;
-                if(isMuted) vol = 0;
+                if (isMuted) vol = 0;
                 if (model.easeInDuration > 0) {
                     const easeIn = Math.min((time - model.start) / model.easeInDuration, 1);
                     vol *= easeIn;
@@ -574,6 +624,60 @@ export class AudioTrackHandler extends TrackHandler {
         }
         this.lastTime = time;
     }
+
+    /** Call to load audio buffer for a specific time in the track. Can be used to preload the timeline audio */
+    loadAudio(time: number, lookAhead: number = 0, lookBehind: number = 0) {
+        let promises: Array<Promise<AudioBuffer | null>> | null = null;
+        const rangeStart = time - lookBehind;
+        const rangeEnd = time + lookAhead;
+        for (const model of this.models) {
+            if (this.isInTimeRange(model, rangeStart, rangeEnd)) {
+                const audio = this.audio[this.models.indexOf(model)];
+                const promise = this.handleAudioLoading(model, audio);
+                if (promise !== null) {
+                    if (promises === null) promises = [];
+                    promises.push(promise);
+                }
+            }
+        }
+        if (promises !== null) {
+            return Promise.all(promises);
+        }
+        return null;
+    }
+
+    private isInTimeRange(model: Models.ClipModel, start: number, end: number) {
+        // Range surrounds clip range
+        if (start <= model.start && end >= model.end) return true;
+        // Range start is in clip range
+        if (start >= model.start && start <= model.end) return true;
+        // Range end is in clip range
+        if (end >= model.start && end <= model.end) return true;
+        return false;
+    }
+
+    private handleAudioLoading(model: Models.ClipModel, audio: THREE.Audio): Promise<AudioBuffer | null> | null {
+        if (!this._audioLoader) {
+            const path = this.getAudioFilePath(model.asset.clip);
+            this._audioLoader = new THREE.AudioLoader();
+            // TODO: maybe we should cache the loaders / buffers here by path
+            if (debug) console.warn("LOAD audio track", path, this.director.sourceId);
+            const loadingPromise = new Promise<AudioBuffer | null>((resolve, _reject) => {
+                this._audioLoader!.load(path,
+                    buffer => {
+                        audio.setBuffer(buffer);
+                        resolve(buffer);
+                    },
+                    undefined,
+                    err => {
+                        console.error("Error loading audio", err);
+                        resolve(null);
+                    });
+            });
+            return loadingPromise;
+        }
+        return null;
+    }
 }
 
 export class SignalTrackHandler extends TrackHandler {