@needle-tools/engine 5.1.0-canary.fbdfce3 → 5.1.0-experimental.03e8105

package/src/engine-components/AudioSource.ts

@@ -4,7 +4,6 @@ import { PositionalAudioHelper } from 'three/examples/jsm/helpers/PositionalAudi
 import { isDevEnvironment } from "../engine/debug/index.js";
 import { Application, ApplicationEvents } from "../engine/engine_application.js";
 import { findObjectOfType } from "../engine/engine_components.js";
-import { Mathf } from "../engine/engine_math.js";
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
 import { AudioListener } from "./AudioListener.js";
@@ -140,12 +139,23 @@ export class AudioSource extends Behaviour {
     get isPlaying(): boolean { return this.sound?.isPlaying ?? false; }
 
     /**
-     * The total duration of the current audio clip in seconds.
+     * The total duration of the currently loaded audio clip in seconds.
      * 
      * @returns Duration in seconds or undefined if no clip is loaded
+     * @remarks For MediaStream clips, duration is not directly available and will return undefined. If the audio clip has not started loading or is still loading, duration may also be undefined until the audio buffer is ready.
      */
     get duration() {
-        return this.sound?.buffer?.duration;
+        if (this.sound?.buffer?.duration) {
+            return this.sound.buffer.duration;
+        }
+        else if (this._audioElement?.duration) {
+            return this._audioElement.duration;
+        }
+        else if (this._loadedClip instanceof MediaStream) {
+            // MediaStream duration is not directly available; return undefined or estimate if possible
+            return undefined;
+        }
+        return undefined;
     }
 
     /**
@@ -200,7 +210,8 @@ export class AudioSource extends Behaviour {
     /**
      * Controls how the audio is positioned in space.
      * Values range from 0 (2D, non-positional) to 1 (fully 3D positioned).
-     * Note: 2D playback is not fully supported in the current implementation.
+     * Internally uses a dual-path audio graph to crossfade between a spatialized (PannerNode)
+     * and a non-spatialized (direct) signal path.
      */
     @serializable()
     get spatialBlend(): number {
@@ -292,12 +303,18 @@ export class AudioSource extends Behaviour {
     // set this from audio context time, used to set clip offset when setting "time" property
     // there is maybe a better way to set a audio clip current time?!
     private _lastClipStartedLoading: string | MediaStream | null = null;
+    private _loadedClip: string | MediaStream | null = null;
     private _audioElement: HTMLAudioElement | null = null;
 
+    // Spatial blend dual-path audio nodes
+    private _entryNode: GainNode | null = null;
+    private _spatialGain: GainNode | null = null;
+    private _bypassGain: GainNode | null = null;
+
     /**
      * Returns the underlying {@link PositionalAudio} object, creating it if necessary.
      * The audio source needs a user interaction to be initialized due to browser autoplay policies.
-     * 
+     *
      * @returns The three.js PositionalAudio object or null if unavailable
      */
     public get Sound(): PositionalAudio | null {
@@ -312,45 +329,12 @@ export class AudioSource extends Behaviour {
             if (listener?.listener) {
                 this.sound = new PositionalAudio(listener.listener);
                 this.gameObject?.add(this.sound);
-
-                // this._listener = listener;
-                // this._originalSoundMatrixWorldFunction = this.sound.updateMatrixWorld;
-                // this.sound.updateMatrixWorld = this._onSoundMatrixWorld;
             }
             else if (debug) console.warn("No audio listener found in scene - can not play audio");
         }
         return this.sound;
     }
 
-    // This is a hacky workaround to get the PositionalAudio behave like a 2D audio source
-    // private _listener: AudioListener | null = null;
-    // private _originalSoundMatrixWorldFunction: Function | null = null;
-    // private _onSoundMatrixWorld = (force: boolean) => {
-    //     if (this._spatialBlend > .05) {
-    //         if (this._originalSoundMatrixWorldFunction) {
-    //             this._originalSoundMatrixWorldFunction.call(this.sound, force);
-    //         }
-    //     }
-    //     else {
-    //         // we use another object's matrix world function (but bound to the positional audio)
-    //         // this is just a little trick to prevent calling the PositionalAudio's updateMatrixWorld function
-    //         this.gameObject.updateMatrixWorld?.call(this.sound, force);
-    //         if (this.sound && this._listener) {
-    //             this.sound.gain.connect(this._listener.listener.getInput());
-    //             // const pos = getTempVector().setFromMatrixPosition(this._listener.gameObject.matrixWorld);
-    //             // const ctx = this.sound.context;
-    //             // const delay = this._listener.listener.timeDelta;
-    //             // const time = ctx.currentTime ;
-    //             // this.sound.panner.positionX.setValueAtTime(pos.x, time);
-    //             // this.sound.panner.positionY.setValueAtTime(pos.y, time);
-    //             // this.sound.panner.positionZ.setValueAtTime(pos.z, time);
-    //             // this.sound.panner.orientationX.setValueAtTime(0, time);
-    //             // this.sound.panner.orientationY.setValueAtTime(0, time);
-    //             // this.sound.panner.orientationZ.setValueAtTime(-1, time);
-    //         }
-    //     }
-    // }
-
     /**
      * Indicates whether the audio source is queued to play when possible.
      * This may be true before user interaction has been registered.
@@ -376,8 +360,10 @@ export class AudioSource extends Behaviour {
         if (this.playOnAwake) this.shouldPlay = true;
 
         if (this.preload) {
-            if (typeof this.clip === "string") {
-                this.audioLoader.load(this.clip, this.createAudio, () => { }, console.error);
+            if (typeof this.clip === "string" && this.clip.length > 0) {
+                this.audioLoader.load(this.clip, this.createAudio, () => { }, e => {
+                    console.error(`[AudioSource] Error preloading audio clip "${this.clip}":`, e);
+                });
             }
         }
     }
@@ -433,9 +419,9 @@ export class AudioSource extends Behaviour {
             this.sound?.setVolume(this.volume);
     }
 
-    private createAudio = (buffer?: AudioBuffer) => {                
-        if(this.destroyed) {
-            if(debug) console.warn("AudioSource destroyed, not creating audio", this.name);
+    private createAudio = (buffer?: AudioBuffer) => {
+        if (this.destroyed) {
+            if (debug) console.warn("AudioSource destroyed, not creating audio", this.name);
             return;
         }
 
@@ -451,26 +437,17 @@ export class AudioSource extends Behaviour {
             sound.stop();
 
         if (buffer) sound.setBuffer(buffer);
+        this._loadedClip = this.clip;
         sound.loop = this._loop;
         if (this.context.application.muted) sound.setVolume(0);
         else sound.setVolume(this.volume);
         sound.autoplay = this.shouldPlay && AudioSource.userInteractionRegistered;
+        this.setupSpatialBlendNodes();
         this.applySpatialDistanceSettings();
 
         if (sound.isPlaying)
             sound.stop();
 
-        // const panner = sound.panner;
-        // panner.coneOuterGain = 1;
-        // sound.setDirectionalCone(360, 360, 1);
-        // const src = sound.context.createBufferSource();
-        // src.buffer = sound.buffer;
-        // src.connect(sound.panner);
-        // src.start(this.audioContext?.currentTime);
-        // const gain = sound.context.createGain();
-        // gain.gain.value = 1 - this.spatialBlend;
-        // src.connect(gain);
-
         // make sure we only play the sound if the user has interacted with the page
         AudioSource.registerWaitForAllowAudio(this.__onAllowAudioCallback);
     }
@@ -479,30 +456,97 @@ export class AudioSource extends Behaviour {
             this.play();
     }
 
+    /**
+     * Sets up the dual-path audio graph for spatial blend crossfading.
+     * Creates two parallel signal paths from source to output:
+     * - 3D path: entry → panner → spatialGain → gain (spatialized)
+     * - 2D path: entry → bypassGain → gain (non-spatialized)
+     */
+    private setupSpatialBlendNodes() {
+        const sound = this.sound;
+        if (!sound || this._entryNode) return;
+
+        const ctx = sound.context;
+        this._entryNode = ctx.createGain();
+        this._spatialGain = ctx.createGain();
+        this._bypassGain = ctx.createGain();
+
+        // Disconnect the default panner → gain connection set up by three.js
+        try { sound.panner.disconnect(sound.gain); } catch { /* not connected */ }
+
+        // 3D path: entry → panner → spatialGain → gain
+        this._entryNode.connect(sound.panner);
+        sound.panner.connect(this._spatialGain);
+        this._spatialGain.connect(sound.gain);
+
+        // 2D path: entry → bypassGain → gain
+        this._entryNode.connect(this._bypassGain);
+        this._bypassGain.connect(sound.gain);
+
+        // Override getOutput so three.js connects source → entryNode instead of panner
+        sound.getOutput = () => this._entryNode! as unknown as PannerNode;
+
+        // Wrap connect() to undo PositionalAudio's redundant panner → gain reconnection
+        const origConnect = sound.connect;
+        sound.connect = function () {
+            origConnect.call(sound);
+            try { sound.panner.disconnect(sound.gain); } catch { /* expected */ }
+            return sound;
+        };
+
+        // Wrap disconnect() to prevent error when panner → gain doesn't exist
+        const origDisconnect = sound.disconnect;
+        sound.disconnect = function () {
+            try { origDisconnect.call(sound); } catch { /* panner → gain already removed */ }
+            return sound;
+        };
+
+        this.updateSpatialBlendGains();
+    }
+
+    /** Updates the spatial/bypass gain values based on the current spatialBlend. */
+    private updateSpatialBlendGains() {
+        if (!this._spatialGain || !this._bypassGain || !this.sound) return;
+        const t = this.sound.context.currentTime;
+        // Use setTargetAtTime for smooth transitions (avoids clicks/pops)
+        this._spatialGain.gain.setTargetAtTime(this._spatialBlend, t, 0.01);
+        this._bypassGain.gain.setTargetAtTime(1 - this._spatialBlend, t, 0.01);
+    }
+
     private applySpatialDistanceSettings() {
         const sound = this.sound;
         if (!sound) return;
         this._needUpdateSpatialDistanceSettings = false;
-        const dist = Mathf.lerp(10 * this._maxDistance / Math.max(0.0001, this.spatialBlend), this._minDistance, this.spatialBlend);
-        if (debug) console.log(this.name, this._minDistance, this._maxDistance, this.spatialBlend, "Ref distance=" + dist);
-        sound.setRefDistance(dist);
-        sound.setMaxDistance(Math.max(0.01, this._maxDistance));
-        // sound.setRolloffFactor(this.spatialBlend);
-        // sound.panner.positionZ.automationRate
+
+        const ref = Math.max(0.0001, this._minDistance);
+        const max = Math.max(0.01, this._maxDistance);
+        if (debug) console.log(this.name, ref, max, this.spatialBlend);
+        sound.setRefDistance(ref);
+        sound.setMaxDistance(max);
 
         // https://developer.mozilla.org/en-US/docs/Web/API/PannerNode/distanceModel
         switch (this.rollOffMode) {
             case AudioRolloffMode.Logarithmic:
-                sound.setDistanceModel('exponential');
+                // Unity's logarithmic rolloff matches Web Audio's 'inverse' model:
+                // gain = refDistance / (refDistance + rolloffFactor * (distance - refDistance))
+                // We compute rolloffFactor so gain reaches ~0.01 (-40dB) at maxDistance,
+                // matching Unity's behavior of effective silence at maxDistance.
+                sound.setDistanceModel('inverse');
+                sound.setRolloffFactor(99 * ref / Math.max(0.001, max - ref));
                 break;
             case AudioRolloffMode.Linear:
+                // gain = 1 - rolloffFactor * (distance - refDistance) / (maxDistance - refDistance)
+                // With rolloffFactor=1 this reaches exactly 0 at maxDistance.
                 sound.setDistanceModel('linear');
+                sound.setRolloffFactor(1);
                 break;
             case AudioRolloffMode.Custom:
                 console.warn("Custom rolloff for AudioSource is not supported: " + this.name);
                 break;
         }
 
+        this.updateSpatialBlendGains();
+
         if (this.spatialBlend > 0) {
             if (debug && !this.helper) {
                 this.helper = new PositionalAudioHelper(sound, sound.getRefDistance());
@@ -519,7 +563,7 @@ export class AudioSource extends Behaviour {
         if (typeof clip === "string") {
             if (debug)
                 console.log(clip);
-            if (clip.endsWith(".mp3") || clip.endsWith(".wav")) {
+            if (clip.endsWith(".mp3") || clip.endsWith(".wav") || clip.endsWith(".ogg") || clip.endsWith(".flac") || clip.endsWith(".aac") || clip.endsWith(".webm")) {
                 if (!this.audioLoader)
                     this.audioLoader = new AudioLoader();
                 this.shouldPlay = true;
@@ -530,9 +574,12 @@ export class AudioSource extends Behaviour {
                 this._lastClipStartedLoading = clip;
                 if (debug)
                     console.log("load audio", clip);
-                const buffer = await this.audioLoader.loadAsync(clip).catch(console.error);
-                if(this.destroyed) return;
-                if(this._lastClipStartedLoading === clip) this._lastClipStartedLoading = null;
+                const buffer = await this.audioLoader.loadAsync(clip).catch(e => {
+                    console.error(`[AudioSource] Error loading audio clip "${clip}":`, e);
+                    return null;
+                });
+                if (this.destroyed) return;
+                if (this._lastClipStartedLoading === clip) this._lastClipStartedLoading = null;
                 if (buffer) this.createAudio(buffer);
             }
             else console.warn("Unsupported audio clip type", clip)
@@ -548,8 +595,9 @@ export class AudioSource extends Behaviour {
      * If no argument is provided, plays the currently assigned clip.
      * 
      * @param clip - Optional audio clip or {@link MediaStream} to play
+     * @returns A promise that resolves when playback starts successfully, or rejects on error
      */
-    play(clip: string | MediaStream | undefined = undefined) {
+    async play(clip: string | MediaStream | undefined = undefined): Promise<boolean> {
         // use audio source's clip when no clip is passed in
         if (!clip && this.clip)
             clip = this.clip;
@@ -557,27 +605,27 @@ export class AudioSource extends Behaviour {
         // We only support strings and media stream
         // TODO: maybe we should return here if an invalid value is passed in
         if (clip !== undefined && typeof clip !== "string" && !(clip instanceof MediaStream)) {
-            if (isDevEnvironment())
-                console.warn("Called play on AudioSource with unknown argument type:", clip + "\nUsing the assigned clip instead:", this.clip)
             // e.g. when a AudioSource.Play is called from SpatialTrigger onEnter this event is called with the TriggerReceiver... to still make this work we *re-use* our already assigned clip. Because otherwise calling `play` would not play the clip...
             clip = this.clip;
+            if (isDevEnvironment()) console.warn("[AudioSource] Called play on AudioSource with unknown argument type:", clip + "\nUsing the assigned clip instead:", this.clip)
         }
 
-        // Check if we need to call load first
-        let needsLoading = !this.sound || (clip && clip !== this.clip);
+        // Load if the sound hasn't been created yet or if the clip changed since last load
+        let needsLoading = !this.sound || (clip && clip !== this._loadedClip);
         if (typeof clip === "string" && !this.audioLoader) needsLoading = true;
         if (clip instanceof MediaStream || typeof clip === "string")
             this.clip = clip;
         if (needsLoading) {
             this.shouldPlay = true;
-            this.onNewClip(clip);
-            return;
+            return this.onNewClip(clip).then(() => true).catch(() => false);
         }
 
         this.shouldPlay = true;
         this._hasEnded = false;
-        if (debug)
-            console.log("play", this.sound?.getVolume(), this.sound);
+
+        if (debug) console.log("[AudioSource] play", this.sound?.getVolume(), this.sound);
+        // If a different clip was passed in, needsLoading above would be true and onNewClip handles it.
+        // This guard prevents double-playing the same already-playing sound (which would throw in three.js).
         if (this.sound && !this.sound.isPlaying) {
             const muted = this.context.application.muted;
             if (muted) this.sound.setVolume(0);
@@ -597,13 +645,17 @@ export class AudioSource extends Behaviour {
                     this.context.domElement.shadowRoot?.append(this._audioElement);
                 this._audioElement.srcObject = this.clip;
                 this._audioElement.autoplay = false;
+                return true;
 
             }
             else {
                 if (this._audioElement) this._audioElement.remove();
                 this.sound.play(muted ? .1 : 0);
+                return true;
             }
         }
+
+        return false;
     }
 
     /**
@@ -611,7 +663,7 @@ export class AudioSource extends Behaviour {
      * Use play() to resume from the paused position.
      */
     pause() {
-        if (debug) console.log("Pause", this);
+        if (debug) console.log("[AudioSource] pause", this);
         this._hasEnded = true;
         this.shouldPlay = false;
         if (this.sound && this.sound.isPlaying && this.sound.source) {
@@ -626,13 +678,13 @@ export class AudioSource extends Behaviour {
      * Unlike pause(), calling play() after stop() will start from the beginning.
      */
     stop() {
-        if (debug) console.log("Pause", this);
+        if (debug) console.log("[AudioSource] stop", this);
         this._hasEnded = true;
         this.shouldPlay = false;
         if (this.sound && this.sound.source) {
             this._lastContextTime = this.sound?.context.currentTime;
             if (debug)
-                console.log(this._lastContextTime)
+                console.log("[AudioSource] lastContextTime", this._lastContextTime);
             this.sound.stop();
         }
         this._audioElement?.remove();
@@ -657,8 +709,7 @@ export class AudioSource extends Behaviour {
 
         if (this.sound && !this.sound.isPlaying && this.shouldPlay && !this._hasEnded) {
             this._hasEnded = true;
-            if (debug)
-                console.log("Audio clip ended", this.clip);
+            if (debug) console.log("[AudioSource] Audio clip ended", this.clip);
             this.dispatchEvent(new CustomEvent("ended", { detail: this }));
         }
 

package/src/engine-components/Collider.ts

@@ -72,12 +72,14 @@ export abstract class Collider extends Behaviour implements ICollider {
      * When `true` the collider will not be used for collision detection but will still trigger events.
      * Trigger colliders can trigger events when other colliders enter their space, without creating a physical response/collision.
      */
+    @validate()
     @serializable()
     isTrigger: boolean = false;
 
     /**
      * The physics material that defines physical properties of the collider such as friction and bounciness.
      */
+    @validate()
     @serializable()
     sharedMaterial?: PhysicsMaterial;
 
@@ -85,15 +87,52 @@ export abstract class Collider extends Behaviour implements ICollider {
      * The layers that this collider belongs to. Used for filtering collision detection.
      * @default [0]
      */
+    @validate()
     @serializable()
     membership: number[] = [0];
 
     /**
      * The layers that this collider will interact with. Used for filtering collision detection.
      */
+    @validate()
     @serializable()
     filter?: number[];
 
+    /**
+     * The density of the collider, used for automatic mass calculation when the attached {@link Rigidbody} has `autoMass` enabled.
+     * Rapier computes mass from density using the real-world formula: `mass = density × volume`.
+     * The volume is derived from the collider shape (sphere, box, capsule, or convex hull).
+     *
+     * Reference values (relative to water = 1.0):
+     * - Wood: 0.5–0.9
+     * - Water: 1.0 (engine default)
+     * - Rubber: 1.2
+     * - Steel: 7.8
+     *
+     * @default undefined — uses the physics engine default of 1.0
+     */
+    @validate()
+    @serializable()
+    density?: number;
+
+    /** @internal */
+    protected _propertiesDirty: boolean = false;
+
+    /** @internal */
+    onValidate(_property?: string) {
+        if (this.activeAndEnabled) {
+            if (_property === "sharedMaterial") {
+                this.updatePhysicsMaterial();
+            }
+            else {
+                this.updateProperties();
+            }
+        }
+        else {
+            this._propertiesDirty = true;
+        }
+    }
+
     /** @internal */
     awake() {
         super.awake();
@@ -197,6 +236,11 @@ export class SphereCollider extends Collider implements ISphereCollider {
         if (!this.context.physics.engine?.setColliderEnabled(this, true)) {
             this.context.physics.engine?.addSphereCollider(this);
         }
+        else if (this._propertiesDirty) {
+            this.updateProperties();
+            this.updatePhysicsMaterial();
+        }
+        this._propertiesDirty = false;
         watchWrite(this.gameObject.scale, this.updateProperties);
     }
 
@@ -208,18 +252,12 @@ export class SphereCollider extends Collider implements ISphereCollider {
         unwatchWrite(this.gameObject.scale, this.updateProperties);
     }
 
-    /**
-     * Updates collider properties when validated in the editor or inspector.
-     */
-    onValidate(): void {
-        this.updateProperties();
-    }
 }
 
 /**
- * BoxCollider represents a box-shaped (cuboid) collision volume.  
- * Most common collider type, efficient for walls, floors, crates, and rectangular objects.  
- * 
+ * BoxCollider represents a box-shaped (cuboid) collision volume.
+ * Most common collider type, efficient for walls, floors, crates, and rectangular objects.
+ *
  * ![](https://cloud.needle.tools/-/media/slYWnXyaxdlrCqu8GP_lFQ.gif)
  *
  * @example Create a floor collider
@@ -284,6 +322,11 @@ export class BoxCollider extends Collider implements IBoxCollider {
         if (!this.context.physics.engine?.setColliderEnabled(this, true)) {
             this.context.physics.engine?.addBoxCollider(this, this.size);
         }
+        else if (this._propertiesDirty) {
+            this.updateProperties();
+            this.updatePhysicsMaterial();
+        }
+        this._propertiesDirty = false;
         watchWrite(this.gameObject.scale, this.updateProperties);
     }
 
@@ -296,14 +339,6 @@ export class BoxCollider extends Collider implements IBoxCollider {
         unwatchWrite(this.gameObject.scale, this.updateProperties);
     }
 
-    /**
-     * Updates collider properties when validated in the editor or inspector.
-     * @internal
-     */
-    onValidate(): void {
-        this.updateProperties();
-    }
-
     /**
      * Automatically fits the collider to the geometry of the object.
      * Sets the size and center based on the object's bounding box.
@@ -379,7 +414,15 @@ export class MeshCollider extends Collider {
         super.onEnable();
         if (!this.context.physics.engine) return;
         // Re-enable existing collider instead of recreating it
-        if (this.context.physics.engine.setColliderEnabled(this, true)) return;
+        if (this.context.physics.engine.setColliderEnabled(this, true)) {
+            if (this._propertiesDirty) {
+                this.updateProperties();
+                this.updatePhysicsMaterial();
+            }
+            this._propertiesDirty = false;
+            return;
+        }
+        this._propertiesDirty = false;
 
         if (!this.sharedMesh?.isMesh) {
             // HACK using the renderer mesh
@@ -475,5 +518,10 @@ export class CapsuleCollider extends Collider {
         if (!this.context.physics.engine?.setColliderEnabled(this, true)) {
             this.context.physics.engine?.addCapsuleCollider(this, this.height, this.radius);
         }
+        else if (this._propertiesDirty) {
+            this.updateProperties();
+            this.updatePhysicsMaterial();
+        }
+        this._propertiesDirty = false;
     }
 }