@needle-tools/engine 4.16.10 → 4.16.11

package/src/engine-components/AudioSource.ts

@@ -1,4 +1,4 @@
-import { AudioLoader, PositionalAudio } from "three";
+import { PositionalAudio } from "three";
 import { PositionalAudioHelper } from 'three/examples/jsm/helpers/PositionalAudioHelper.js';
 
 import { isDevEnvironment } from "../engine/debug/index.js";
@@ -137,7 +137,13 @@ export class AudioSource extends Behaviour {
      * 
      * @returns True if the audio is playing, false otherwise
      */
-    get isPlaying(): boolean { return this.sound?.isPlaying ?? false; }
+    get isPlaying(): boolean {
+        // For the media-element path (file-URL string clips) three.js does NOT maintain
+        // sound.isPlaying (hasPlaybackControl is false once a media source is set), so the
+        // <audio> element is the source of truth.
+        if (this._usesMediaElementSource && this._audioElement) return !this._audioElement.paused && !this._audioElement.ended;
+        return this.sound?.isPlaying ?? false;
+    }
 
     /**
      * The total duration of the current audio clip in seconds.
@@ -145,6 +151,9 @@ export class AudioSource extends Behaviour {
      * @returns Duration in seconds or undefined if no clip is loaded
      */
     get duration() {
+        // Media-element path is the primary source of truth for file-URL string clips.
+        const elDuration = this._usesMediaElementSource ? this._audioElement?.duration : undefined;
+        if (elDuration && isFinite(elDuration)) return elDuration;
         return this.sound?.buffer?.duration;
     }
 
@@ -154,14 +163,14 @@ export class AudioSource extends Behaviour {
      */
     get time01() {
         const duration = this.duration;
-        if (duration && this.sound) {
-            return this.sound?.context.currentTime / duration;
+        if (duration) {
+            return this.time / duration;
         }
         return 0;
     }
     set time01(val: number) {
         const duration = this.duration;
-        if (duration && this.sound) {
+        if (duration) {
             this.time = val * duration;
         }
     }
@@ -170,8 +179,17 @@ export class AudioSource extends Behaviour {
      * The current playback position in seconds.
      * Can be set to seek to a specific time in the audio.
      */
-    get time(): number { return this.sound?.source ? (this.sound.source?.context.currentTime - this._lastContextTime + this.sound.offset) : 0; }
+    get time(): number {
+        // Media-element path: the element's currentTime is authoritative.
+        if (this._usesMediaElementSource && this._audioElement) return this._audioElement.currentTime;
+        return this.sound?.source ? (this.sound.source?.context.currentTime - this._lastContextTime + this.sound.offset) : 0;
+    }
     set time(val: number) {
+        // Media-element path: seek directly on the element (sound.offset is not used for media nodes).
+        if (this._usesMediaElementSource && this._audioElement) {
+            this._audioElement.currentTime = val;
+            return;
+        }
         if (this.sound) {
             if (val === this.sound.offset) return;
             const wasPlaying = this.isPlaying;
@@ -189,12 +207,18 @@ export class AudioSource extends Behaviour {
      */
     @serializable()
     get loop(): boolean {
+        // Media-element path: the element holds the authoritative loop flag.
+        if (this._usesMediaElementSource && this._audioElement) {
+            this._loop = this._audioElement.loop;
+            return this._loop;
+        }
         if (this.sound) this._loop = this.sound.getLoop();
         return this._loop;
     }
     set loop(val: boolean) {
         this._loop = val;
-        if (this.sound) this.sound.setLoop(val);
+        if (this._usesMediaElementSource && this._audioElement) this._audioElement.loop = val;
+        else if (this.sound) this.sound.setLoop(val);
     }
 
     /**
@@ -268,9 +292,13 @@ export class AudioSource extends Behaviour {
      */
     @serializable()
     set pitch(val: number) {
-        if (this.sound) this.sound.setPlaybackRate(val);
+        // Media-element path: playbackRate on the element (sound.setPlaybackRate is a no-op once
+        // hasPlaybackControl is false).
+        if (this._usesMediaElementSource && this._audioElement) this._audioElement.playbackRate = val;
+        else if (this.sound) this.sound.setPlaybackRate(val);
     }
     get pitch(): number {
+        if (this._usesMediaElementSource && this._audioElement) return this._audioElement.playbackRate;
         return this.sound ? this.sound.getPlaybackRate() : 1;
     }
 
@@ -287,12 +315,21 @@ export class AudioSource extends Behaviour {
     private sound: PositionalAudio | null = null;
     private helper: PositionalAudioHelper | null = null;
     private wasPlaying = false;
-    private audioLoader: AudioLoader | null = null;
     private shouldPlay: boolean = false;
     // set this from audio context time, used to set clip offset when setting "time" property
     // there is maybe a better way to set a audio clip current time?!
-    private _lastClipStartedLoading: string | MediaStream | null = null;
+    private _loadedClip: string | MediaStream | null = null;
     private _audioElement: HTMLAudioElement | null = null;
+    /**
+     * True when {@link _audioElement} is a file-URL element routed through
+     * `sound.setMediaElementSource()` — the iOS-survivable string-clip path. For this path ALL
+     * playback control (play/pause/stop/loop/pitch/time/isPlaying) goes through the element,
+     * because three.js sets `hasPlaybackControl = false` once a media source is attached.
+     * It is `false` for the distinct MediaStream path.
+     */
+    private _usesMediaElementSource: boolean = false;
+    /** Dedicated gain between the media source and the panner, used for click-free pause/resume fades. */
+    private _fadeNode: GainNode | null = null;
 
     /**
      * Returns the underlying {@link PositionalAudio} object, creating it if necessary.
@@ -368,16 +405,42 @@ export class AudioSource extends Behaviour {
         return this.sound?.context;
     }
 
+    /**
+     * Resumes the shared AudioContext if it is currently suspended/interrupted and the page is
+     * visible. No-op when already running or when resuming is not allowed (resuming a hidden
+     * context throws iOS `InvalidStateError: Failed to start the audio device`).
+     */
+    private ensureContextResumed() {
+        const ctx = this.audioContext;
+        if (!ctx) return;
+        const visibility = typeof document !== "undefined" ? document.visibilityState : "visible";
+        if (visibility !== "visible") return;
+        // Resume even when ctx.state reports "running": on iOS an auto-resume that happened outside a
+        // user gesture can leave the context "running" while the audio hardware stays off. play() is
+        // user-initiated, so this resume() runs inside the gesture and re-enables real output.
+        ctx.resume().catch(e => { if (debug) console.warn("[AudioSource] Failed to resume AudioContext:", e); });
+    }
+
     /** @internal */
     awake() {
         if (debug) console.log("[AudioSource]", this);
 
-        this.audioLoader = new AudioLoader();
         if (this.playOnAwake) this.shouldPlay = true;
 
         if (this.preload) {
-            if (typeof this.clip === "string") {
-                this.audioLoader.load(this.clip, this.createAudio, () => { }, console.error);
+            if (typeof this.clip === "string" && this.clip.length > 0) {
+                // Preload = create the <audio> element and wire it into the graph, but do NOT play.
+                // Wiring needs the PositionalAudio (Sound), which needs an AudioListener that only
+                // exists after a user gesture — so defer when not ready.
+                if (this.Sound) {
+                    this.createAudioFromElement();
+                }
+                else {
+                    AudioSource.registerWaitForAllowAudio(() => {
+                        if (this.destroyed || this.enabled === false) return;
+                        if (typeof this.clip === "string" && this.clip.length > 0) this.createAudioFromElement();
+                    });
+                }
             }
         }
     }
@@ -411,15 +474,22 @@ export class AudioSource extends Behaviour {
         switch (document.visibilityState) {
             case "hidden":
                 if (this.playInBackground === false || DeviceUtilities.isMobileDevice()) {
-                    this.wasPlaying = this.isPlaying;
-                    if (this.isPlaying) {
+                    const wasPlaying = this.isPlaying;
+                    if (wasPlaying) {
                         this.pause();
                     }
+                    // pause() clears wasPlaying so a *user-initiated* pause is not auto-resumed on
+                    // the next foreground. This pause is automatic (page hidden / device locked), so
+                    // re-assert the resume intent AFTER pause() — otherwise we never resume on unlock.
+                    this.wasPlaying = wasPlaying;
                 }
                 break;
             case "visible":
                 if (debug) console.log("visible", this.enabled, this.playOnAwake, !this.isPlaying, AudioSource.userInteractionRegistered, this.wasPlaying);
-                if (this.enabled && this.playOnAwake && !this.isPlaying && AudioSource.userInteractionRegistered && this.wasPlaying) {
+                // Resume anything that was actively playing when the page was hidden (not just
+                // playOnAwake sources). wasPlaying is only set when this source was playing at hide
+                // time, so script-started clips resume too.
+                if (this.enabled && !this.isPlaying && AudioSource.userInteractionRegistered && this.wasPlaying) {
                     this.play();
                 }
                 break;
@@ -433,44 +503,117 @@ export class AudioSource extends Behaviour {
             this.sound?.setVolume(this.volume);
     }
 
-    private createAudio = (buffer?: AudioBuffer) => {                
-        if(this.destroyed) {
-            if(debug) console.warn("AudioSource destroyed, not creating audio", this.name);
+    /**
+     * Sets up playback of a file-URL string clip through an `HTMLAudioElement` routed into the Web
+     * Audio graph via `sound.setMediaElementSource()`.
+     *
+     * Why a media element instead of a decoded `AudioBuffer`: on iOS Safari an
+     * `AudioBufferSourceNode`'s output is torn down after a device lock / audio-session interruption
+     * and cannot be reliably revived. A `MediaElementAudioSourceNode` survives because the playing
+     * `<audio>` element keeps iOS's media session alive, so the shared AudioContext resumes cleanly
+     * on unlock. This mirrors three.js's `webaudio_orientation` example.
+     *
+     * Does NOT start playback — it only creates/wires the element so a later `.play()` is cheap.
+     */
+    private createAudioFromElement() {
+        if (this.destroyed) {
+            if (debug) console.warn("AudioSource destroyed, not creating audio", this.name);
             return;
         }
-
-        if (debug) console.log("AudioBuffer finished loading", buffer);
-
         const sound = this.Sound;
         if (!sound) {
             if (debug) console.warn("Failed getting sound?", this.name);
             return;
         }
 
-        if (sound.isPlaying)
-            sound.stop();
+        // Already wired for the current clip — re-running setMediaElementSource on the same element
+        // throws InvalidStateError, so just keep volume/mute in sync and bail.
+        if (this._audioElement && this._usesMediaElementSource && this._loadedClip === this.clip) {
+            if (this.context.application.muted) sound.setVolume(0);
+            else sound.setVolume(this.volume);
+            return;
+        }
+
+        // (re)create the element when the clip changed since the element was wired.
+        if (this._audioElement && this._loadedClip !== this.clip) {
+            this._audioElement.pause();
+            this._audioElement = null;
+            this._usesMediaElementSource = false;
+        }
+        if (!this._audioElement) {
+            const el = new Audio();
+            el.crossOrigin = "anonymous";
+            el.preload = "auto";
+            el.src = this.clip as string;
+            this._audioElement = el;
+        }
+        this._audioElement!.loop = this._loop;
+
+        // IMPORTANT: build the fade node FIRST so getOutput() returns it, THEN route the media
+        // element source in. setMediaElementSource() immediately calls sound.connect()
+        // (source → getOutput()), so the fade node and the getOutput override must already exist.
+        this.setupFadeNode();
+        // Only connect once per element — createMediaElementSource() throws if the same element is
+        // passed twice.
+        if (!this._usesMediaElementSource) {
+            sound.setMediaElementSource(this._audioElement!);
+            this._usesMediaElementSource = true;
+        }
+        this._loadedClip = this.clip;
 
-        if (buffer) sound.setBuffer(buffer);
-        sound.loop = this._loop;
         if (this.context.application.muted) sound.setVolume(0);
         else sound.setVolume(this.volume);
-        sound.autoplay = this.shouldPlay && AudioSource.userInteractionRegistered;
         this.applySpatialDistanceSettings();
+    }
 
-        if (sound.isPlaying)
-            sound.stop();
+    /**
+     * Inserts a dedicated gain node between the media source and the panner so pause/resume can fade
+     * click-free without touching `sound.gain` (which carries volume/mute). three.js connects the
+     * source to `getOutput()` inside `setMediaElementSource()`, so we override it to feed our fade
+     * node, then route fade → panner (→ gain → listener, both already wired by PositionalAudio).
+     */
+    private setupFadeNode() {
+        const sound = this.sound;
+        if (!sound || this._fadeNode) return;
+        this._fadeNode = sound.context.createGain();
+        this._fadeNode.connect(sound.panner);
+        sound.getOutput = () => this._fadeNode! as unknown as PannerNode;
+    }
 
-        // const panner = sound.panner;
-        // panner.coneOuterGain = 1;
-        // sound.setDirectionalCone(360, 360, 1);
-        // const src = sound.context.createBufferSource();
-        // src.buffer = sound.buffer;
-        // src.connect(sound.panner);
-        // src.start(this.audioContext?.currentTime);
-        // const gain = sound.context.createGain();
-        // gain.gain.value = 1 - this.spatialBlend;
-        // src.connect(gain);
+    /**
+     * Smoothly ramp the fade gain to `target` (1 = audible, 0 = silent) over a few ms. A short fade —
+     * not a hard 0/1 gain change and not a connect/disconnect — avoids the click on pause/resume. On
+     * iOS the MediaElementAudioSourceNode keeps feeding the graph after element.pause() following an
+     * interruption, so fading this gain to 0 is what actually silences a "paused" media clip.
+     */
+    private fadeGain(target: number, seconds = 0.02) {
+        if (!this._fadeNode || !this.sound) return;
+        const ctx = this.sound.context;
+        const g = this._fadeNode.gain;
+        g.cancelScheduledValues(ctx.currentTime);
+        g.setValueAtTime(g.value, ctx.currentTime);
+        g.linearRampToValueAtTime(target, ctx.currentTime + seconds);
+    }
 
+    /**
+     * Sets up the {@link PositionalAudio} graph for a MediaStream clip (file-URL string clips go
+     * through {@link createAudioFromElement}). The stream node itself is attached in {@link play}
+     * via `setMediaStreamSource`.
+     */
+    private createAudio = () => {
+        if (this.destroyed) {
+            if (debug) console.warn("AudioSource destroyed, not creating audio", this.name);
+            return;
+        }
+        const sound = this.Sound;
+        if (!sound) {
+            if (debug) console.warn("Failed getting sound?", this.name);
+            return;
+        }
+        this._loadedClip = this.clip;
+        if (this.context.application.muted) sound.setVolume(0);
+        else sound.setVolume(this.volume);
+        this.applySpatialDistanceSettings();
         // make sure we only play the sound if the user has interacted with the page
         AudioSource.registerWaitForAllowAudio(this.__onAllowAudioCallback);
     }
@@ -520,20 +663,20 @@ export class AudioSource extends Behaviour {
             if (debug)
                 console.log(clip);
             if (clip.endsWith(".mp3") || clip.endsWith(".wav")) {
-                if (!this.audioLoader)
-                    this.audioLoader = new AudioLoader();
                 this.shouldPlay = true;
-                if (this._lastClipStartedLoading === clip) {
-                    if (debug) console.log("Is currently loading:", this._lastClipStartedLoading, this)
-                    return;
+                // Route the file URL through an <audio> element (survives iOS lock — see
+                // createAudioFromElement). This creates/wires the element without playing.
+                this.createAudioFromElement();
+                // Start playback through the element (NOT this.sound — hasPlaybackControl is false
+                // for a media-element source). Covers the already-interacted case directly.
+                if (this.shouldPlay && AudioSource.userInteractionRegistered) {
+                    this.ensureContextResumed();
+                    this._hasEnded = false;
+                    this._audioElement?.play().catch(e => {
+                        if (debug) console.warn(`[AudioSource] Failed to play audio clip "${clip}":`, e);
+                    });
+                    this.fadeGain(1);
                 }
-                this._lastClipStartedLoading = clip;
-                if (debug)
-                    console.log("load audio", clip);
-                const buffer = await this.audioLoader.loadAsync(clip).catch(console.error);
-                if(this.destroyed) return;
-                if(this._lastClipStartedLoading === clip) this._lastClipStartedLoading = null;
-                if (buffer) this.createAudio(buffer);
             }
             else console.warn("Unsupported audio clip type", clip)
         }
@@ -563,9 +706,11 @@ export class AudioSource extends Behaviour {
             clip = this.clip;
         }
 
-        // Check if we need to call load first
-        let needsLoading = !this.sound || (clip && clip !== this.clip);
-        if (typeof clip === "string" && !this.audioLoader) needsLoading = true;
+        // Load if the sound hasn't been created yet or if the clip changed since last load.
+        let needsLoading = !this.sound || (clip && clip !== this._loadedClip);
+        // For string clips the <audio> element must additionally be wired (createAudioFromElement);
+        // if it isn't yet (e.g. preload was off, or the element was torn down), force a (re)load.
+        if (typeof clip === "string" && (!this._audioElement || !this._usesMediaElementSource)) needsLoading = true;
         if (clip instanceof MediaStream || typeof clip === "string")
             this.clip = clip;
         if (needsLoading) {
@@ -578,15 +723,21 @@ export class AudioSource extends Behaviour {
         this._hasEnded = false;
         if (debug)
             console.log("play", this.sound?.getVolume(), this.sound);
-        if (this.sound && !this.sound.isPlaying) {
+        // Use the element-aware isPlaying getter so we don't re-trigger an already-playing media clip.
+        if (this.sound && !this.isPlaying) {
+            // On iOS the shared context can be suspended/interrupted (lock, backgrounding, a call) —
+            // resume it here (we are inside a user gesture) so playback doesn't silently depend on
+            // the global resume handler running first. See engine_audio.ts.
+            this.ensureContextResumed();
             const muted = this.context.application.muted;
             if (muted) this.sound.setVolume(0);
             this.gameObject?.add(this.sound);
 
             if (this.clip instanceof MediaStream) {
 
-                // We have to set the audio element source to the mediastream as well
-                // otherwise it will not play for some reason...
+                // Distinct from the file-URL media-element path: a MediaStream uses
+                // setMediaStreamSource + an element holding srcObject.
+                this._usesMediaElementSource = false;
                 this.sound.setMediaStreamSource(this.clip);
 
                 if (!this._audioElement) {
@@ -600,8 +751,13 @@ export class AudioSource extends Behaviour {
 
             }
             else {
-                if (this._audioElement) this._audioElement.remove();
-                this.sound.play(muted ? .1 : 0);
+                // File-URL media-element path: playback control is on the element, not this.sound.
+                // Resume from the (frozen) position and fade the gain back in so the resume is
+                // click-free.
+                this._audioElement?.play().catch(e => {
+                    if (debug) console.warn("[AudioSource] Failed to play audio element:", e);
+                });
+                this.fadeGain(1);
             }
         }
     }
@@ -611,13 +767,26 @@ export class AudioSource extends Behaviour {
      * Use play() to resume from the paused position.
      */
     pause() {
-        if (debug) console.log("Pause", this);
+        if (debug) console.log("[AudioSource] pause", this);
         this._hasEnded = true;
         this.shouldPlay = false;
+        // A user-initiated pause must not be revived by the visibility handler on the next
+        // foreground (it replays sources whose `wasPlaying` is set).
+        this.wasPlaying = false;
+        if (this._usesMediaElementSource) {
+            // File-URL media-element path. On iOS the MediaElementAudioSourceNode keeps emitting into
+            // the graph after element.pause() once the audio session has been interrupted — pausing
+            // the element does NOT silence it. So fade the gain to 0 (click-free silence), then
+            // freeze the element's position. play() fades it back in.
+            this.fadeGain(0);
+            this._audioElement?.pause();
+            return;
+        }
         if (this.sound && this.sound.isPlaying && this.sound.source) {
             this._lastContextTime = this.sound?.context.currentTime;
             this.sound.pause();
         }
+        // MediaStream / legacy path: drop the stream element as before.
         this._audioElement?.remove();
     }
 
@@ -626,15 +795,25 @@ export class AudioSource extends Behaviour {
      * Unlike pause(), calling play() after stop() will start from the beginning.
      */
     stop() {
-        if (debug) console.log("Pause", this);
+        if (debug) console.log("[AudioSource] stop", this);
         this._hasEnded = true;
         this.shouldPlay = false;
+        this.wasPlaying = false;
+        if (this._usesMediaElementSource) {
+            // True stop: fade to silence (element.pause() alone doesn't silence it on iOS after an
+            // interruption), pause, and rewind to the start.
+            this.fadeGain(0);
+            this._audioElement?.pause();
+            if (this._audioElement) this._audioElement.currentTime = 0;
+            return;
+        }
         if (this.sound && this.sound.source) {
             this._lastContextTime = this.sound?.context.currentTime;
             if (debug)
-                console.log(this._lastContextTime)
+                console.log("[AudioSource] lastContextTime", this._lastContextTime);
             this.sound.stop();
         }
+        // MediaStream / legacy path: drop the stream element as before.
         this._audioElement?.remove();
     }
 
@@ -655,7 +834,9 @@ export class AudioSource extends Behaviour {
             this.applySpatialDistanceSettings();
         }
 
-        if (this.sound && !this.sound.isPlaying && this.shouldPlay && !this._hasEnded) {
+        // Use the element-aware isPlaying getter, NOT sound.isPlaying: for a media-element source
+        // three.js leaves sound.isPlaying permanently false, which would fire "ended" every frame.
+        if (this.sound && !this.isPlaying && this.shouldPlay && !this._hasEnded) {
             this._hasEnded = true;
             if (debug)
                 console.log("Audio clip ended", this.clip);

package/src/engine-components/CameraUtils.ts

@@ -46,7 +46,7 @@ ContextRegistry.registerCallback(ContextEvent.MissingCamera, (evt) => {
         // Don't set the background color if the user set a background color in the <needle-engine> element
         if (!evt.context.domElement.getAttribute("background-color")) {
             let backgroundColor = "#efefef";
-            if (typeof window !== undefined && (window.matchMedia('(prefers-color-scheme: dark)').matches)) {
+            if (typeof window !== "undefined" && (window.matchMedia('(prefers-color-scheme: dark)').matches)) {
                 backgroundColor = "#1f1f1f";
             }
             scene.background = new Color(backgroundColor); // dont set it on the camera because this might be controlled from "background-color" attribute which is set on the scene directly. If the camera has a background color, it will override the scene's background color