@needle-tools/engine 5.1.0-alpha.6 → 5.1.0-canary.02ccb45

package/src/engine-components/VideoPlayer.ts

@@ -207,16 +207,13 @@ export class VideoPlayer extends Behaviour {
      */
     get isPlaying(): boolean {
         const video = this._videoElement;
-        if (video) {
-            if (video.currentTime > 0 && !video.paused && !video.ended
-                && video.readyState > video.HAVE_CURRENT_DATA)
-                return true;
-            else if (video.srcObject) {
-                const stream = video.srcObject as MediaStream;
-                if (stream.active) return true;
-            }
+        if (!video) return false;
+        if (video.paused || video.ended) return false;
+        if (video.srcObject) {
+            const stream = video.srcObject as MediaStream;
+            return stream.active;
         }
-        return false;
+        return video.currentTime > 0 && video.readyState > video.HAVE_CURRENT_DATA;
     }
 
     get crossOrigin(): string | null {
@@ -430,26 +427,52 @@ export class VideoPlayer extends Behaviour {
 
     private _playErrors: number = 0;
 
-    /** start playing the video source */
-    play() {
+    /**
+     * Plays the assigned video clip, URL, or MediaStream.
+     * If a `clip` argument is passed, it is used as the new video source (mirroring {@link AudioSource.play}).
+     *
+     * @param clip - Optional video URL string or {@link MediaStream} to play. If omitted, plays the currently assigned source.
+     * @returns A promise that resolves to `true` when playback was successfully started, or `false` on error.
+     */
+    async play(clip?: string | MediaStream): Promise<boolean> {
+        // Defensive: if called from an event handler with a non-string/non-MediaStream argument
+        // (e.g. SpatialTrigger.onEnter passes a receiver object), ignore the arg and fall back to the assigned source.
+        // Same pattern as AudioSource.play.
+        if (clip !== undefined && typeof clip !== "string" && !(clip instanceof MediaStream)) {
+            if (isDevEnvironment()) console.warn("[VideoPlayer] Called play with unknown argument type. Using assigned source instead.", clip);
+            clip = undefined;
+        }
+
+        if (typeof clip === "string") {
+            this.setClipURL(clip);
+        }
+        else if (clip instanceof MediaStream) {
+            this.setVideo(clip);
+        }
+
         if (!this._videoElement) this.create(false);
         if (!this._videoElement) {
             if (debug) console.warn("Can not play: no video element found", this);
-            return
+            return false;
         }
-        if (this._isPlaying && !this._videoElement?.ended && !this._videoElement?.paused) return;
+        if (this._isPlaying && !this._videoElement?.ended && !this._videoElement?.paused) return true;
         this._isPlaying = true;
         if (!this._receivedInput) this._videoElement.muted = true;
         this.handleBeginPlaying(false);
 
         if (this.shouldUseM3U) {
             this.ensureM3UCanBePlayed();
-            return;
+            return true;
         }
 
         if (debug) console.log("Video Play()", this.clip, this._videoElement, this.time);
         this._videoElement.currentTime = this.time;
-        this._videoElement.play().catch(err => {
+        try {
+            await this._videoElement.play();
+            if (debug) console.log("play", this._videoElement, this.time);
+            return true;
+        }
+        catch (err: any) {
             if (this._playErrors++ < 10) console.error(err);
             else if (this._playErrors === 10) console.error("Multiple errors playing video, further errors will be suppressed. Use 'debugvideo' param to see all errors.");
             // https://developer.chrome.com/blog/play-request-was-interrupted/
@@ -459,8 +482,8 @@ export class VideoPlayer extends Behaviour {
                 if (this._isPlaying && !this.destroyed && this.activeAndEnabled)
                     this.play();
             }, 1000);
-        });
-        if (debug) console.log("play", this._videoElement, this.time);
+            return false;
+        }
     }
 
     /**

package/src/engine-components/Voip.ts

@@ -36,9 +36,9 @@ const debugParam = getParam("debugvoip");
  * voip.createMenuButton = true;
  *
  * // Manual control
- * voip.connect();    // Start sending audio
- * voip.disconnect(); // Stop sending
- * voip.setMuted(true); // Mute microphone
+ * voip.connect();    // Start sending your microphone
+ * voip.disconnect(); // Stop sending your microphone
+ * voip.setMuted(true); // Mute incoming audio (silence other users)
  * ```
  *
  * @summary Voice over IP for networked audio communication
@@ -84,9 +84,13 @@ export class Voip extends Behaviour {
     @serializable()
     get volume(): number { return this._volume; }
     set volume(val: number) {
-        this._volume = val;
+        // HTMLMediaElement.volume throws IndexSizeError outside [0,1] — clamp before assigning.
+        // Reject NaN so we don't poison _volume and break serialization.
+        if (Number.isNaN(val)) return;
+        const clamped = Math.max(0, Math.min(1, val));
+        this._volume = clamped;
         for (const audio of this._incomingStreams.values()) {
-            audio.volume = val;
+            audio.volume = clamped;
         }
     }
 
@@ -119,11 +123,11 @@ export class Voip extends Behaviour {
     }
 
     /**
-     * Threshold for speaking detection (0–255). When a user's audio amplitude exceeds this,
-     * they are considered "speaking". Default is 30.
+     * Normalized amplitude threshold for speaking detection (0–1). When a user's average
+     * audio amplitude exceeds this, they are considered "speaking". Default is 0.1.
      */
     @serializable()
-    speakingThreshold: number = 30;
+    speakingThreshold: number = 0.1;
 
     /**
      * Event fired when a user's speaking state changes.
@@ -133,7 +137,11 @@ export class Voip extends Behaviour {
     onSpeakingChanged: EventList = new EventList();
 
     private _speakingStates = new Map<string, boolean>();
-    private _analysers = new Map<string, { analyser: AnalyserNode, data: Uint8Array, context: AudioContext }>();
+    private _analysers = new Map<string, { source: MediaStreamAudioSourceNode, analyser: AnalyserNode, data: Uint8Array }>();
+    // Single shared AudioContext for all remote-user analysers. Browsers cap live AudioContexts at ~6 per tab,
+    // so one-per-user would break voice rooms of 7+ participants. Lazily created on first setupAnalyser.
+    private _sharedAudioContext?: AudioContext;
+    private _lastSpeakingPollMs = 0;
 
     private _net?: NetworkedStreams;
     private _menubutton?: HTMLElement;
@@ -142,12 +150,12 @@ export class Voip extends Behaviour {
     awake() {
         if (debugParam) this.debug = true;
         if (this.debug) {
-            console.log("VOIP debugging: press 'v' to toggle mute or 'c' to toggle connect/disconnect");
+            console.log("VOIP debugging: press 'v' to toggle incoming mute, 'c' to toggle connect/disconnect");
             window.addEventListener("keydown", async (evt) => {
                 const key = evt.key.toLowerCase();
                 switch (key) {
                     case "v":
-                        console.log("MUTE?", !this.isMuted)
+                        console.log("VOIP: toggle incoming mute → ", !this.isMuted);
                         this.setMuted(!this.isMuted);
                         break;
                     case "c":
@@ -156,15 +164,6 @@ export class Voip extends Behaviour {
                         break;
                 }
             });
-            // mute unfocused
-            window.addEventListener("blur", () => {
-                console.log("VOIP: MUTE ON BLUR")
-                this.setMuted(true);
-            });
-            window.addEventListener("focus", () => {
-                console.log("VOIP: UNMUTE ON FOCUS")
-                this.setMuted(false);
-            });
         }
     }
 
@@ -201,10 +200,11 @@ export class Voip extends Behaviour {
         this.onEnabledChanged();
         this.updateButton();
         window.removeEventListener("visibilitychange", this.onVisibilityChanged);
-        // Clean up analysers
+        // Clean up analysers and the shared AudioContext (lazily recreated on next setupAnalyser).
         for (const userId of [...this._analysers.keys()]) {
             this.cleanupAnalyser(userId);
         }
+        this.closeSharedAudioContext();
     }
 
     /** @internal */
@@ -215,6 +215,7 @@ export class Voip extends Behaviour {
         for (const userId of [...this._analysers.keys()]) {
             this.cleanupAnalyser(userId);
         }
+        this.closeSharedAudioContext();
         for (const incoming of this._incomingStreams.values()) {
             disposeStream(incoming.srcObject as MediaStream);
         }
@@ -225,6 +226,10 @@ export class Voip extends Behaviour {
     /** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */
     private _allowSending = true;
     private _outputStream: MediaStream | null = null;
+    // Tracks an in-flight connect() so concurrent callers coalesce onto the same promise
+    // instead of each running getUserMedia in parallel (which would leak a MediaStream and
+    // briefly transmit then kill the first acquired stream).
+    private _connectInFlight?: Promise<boolean>;
 
     /**
      * @returns true if the component is currently sending audio
@@ -233,7 +238,18 @@ export class Voip extends Behaviour {
 
 
     /** Start sending audio. */
-    async connect(audioSource?: MediaTrackConstraints) {
+    connect(audioSource?: MediaTrackConstraints): Promise<boolean> {
+        // Coalesce concurrent callers. Without this, two near-simultaneous connect() calls
+        // each call getUserMedia in parallel, the first stream gets disposed mid-broadcast
+        // by the second, and a MediaStream leaks.
+        if (this._connectInFlight) return this._connectInFlight;
+        this._connectInFlight = this._connectImpl(audioSource).finally(() => {
+            this._connectInFlight = undefined;
+        });
+        return this._connectInFlight;
+    }
+
+    private async _connectImpl(audioSource?: MediaTrackConstraints): Promise<boolean> {
         if (!this._net) {
             console.error("Cannot connect to voice chat - NetworkedStreams not initialized. Make sure the component is enabled before calling this method.");
             return false;
@@ -281,27 +297,25 @@ export class Voip extends Behaviour {
     }
 
     /**
-     * Mute or unmute the audio stream (this will only mute incoming streams and not mute your own microphone. Use disconnect() to mute your own microphone)
+     * Mute or unmute the audio you hear from other users (incoming streams).
+     * This does NOT mute your own microphone — use {@link disconnect} to stop sending your microphone.
      */
     setMuted(mute: boolean) {
-        const audio = this._outputStream?.getAudioTracks();
-        if (audio) {
-            for (const track of audio) {
-                track.enabled = !mute
-            }
+        for (const audio of this._incomingStreams.values()) {
+            audio.muted = mute;
         }
     }
 
-    /** Returns true if the audio stream is currently muted */
+    /**
+     * Returns true if incoming audio is currently muted (you can't hear other users).
+     * When there are no incoming streams, returns false.
+     */
     get isMuted() {
-        if (this._outputStream === null) return false;
-        const audio = this._outputStream?.getAudioTracks();
-        if (audio) {
-            for (const track of audio) {
-                if (!track.enabled) return true;
-            }
+        if (this._incomingStreams.size === 0) return false;
+        for (const audio of this._incomingStreams.values()) {
+            if (!audio.muted) return false;
         }
-        return false;
+        return true;
     }
 
     private async updateButton() {
@@ -381,7 +395,7 @@ export class Voip extends Behaviour {
                 .catch((err) => {
                     console.warn("VOIP failed getting audio stream", err);
                     return null;
-                });;
+                });
         }
 
         const stream = await getUserMedia(audio);
@@ -397,7 +411,13 @@ export class Voip extends Behaviour {
             if (nonBuiltInAudioSource) {
                 const constraints = Object.assign({}, audio);
                 constraints.deviceId = nonBuiltInAudioSource.deviceId;
-                return await getUserMedia(constraints);
+                const externalStream = await getUserMedia(constraints);
+                if (externalStream) {
+                    // Release the built-in mic stream we grabbed first — otherwise its tracks stay live.
+                    disposeStream(stream);
+                    return externalStream;
+                }
+                // External device acquisition failed — keep the original stream rather than returning null.
             }
         }
 
@@ -431,20 +451,25 @@ export class Voip extends Behaviour {
     update() {
         // Only run speaking detection if someone is listening
         if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return;
+        // Rate-limit analysis to ~10Hz. Speaking-state is a coarse UI signal; running FFT per
+        // remote user every frame (60Hz) is wasteful and scales linearly with participant count.
+        const now = performance.now();
+        if (now - this._lastSpeakingPollMs < 100) return;
+        this._lastSpeakingPollMs = now;
 
         for (const [userId, info] of this._analysers) {
             info.analyser.getByteFrequencyData(info.data as Uint8Array<ArrayBuffer>);
-            // Average amplitude
+            // Average amplitude normalized to 0–1
             let sum = 0;
             for (let i = 0; i < info.data.length; i++) sum += info.data[i];
-            const avg = sum / info.data.length;
+            const volume = sum / info.data.length / 255;
 
             const wasSpeaking = this._speakingStates.get(userId) ?? false;
-            const isSpeaking = avg > this.speakingThreshold;
+            const isSpeaking = volume > this.speakingThreshold;
 
             if (isSpeaking !== wasSpeaking) {
                 this._speakingStates.set(userId, isSpeaking);
-                this.onSpeakingChanged.invoke({ userId, isSpeaking, volume: avg / 255 });
+                this.onSpeakingChanged.invoke({ userId, isSpeaking, volume });
             }
         }
     }
@@ -453,13 +478,16 @@ export class Voip extends Behaviour {
         // Only set up if someone is listening or might listen
         if (this._analysers.has(userId)) return;
         try {
-            const audioCtx = new AudioContext();
-            const source = audioCtx.createMediaStreamSource(stream);
-            const analyser = audioCtx.createAnalyser();
+            if (!this._sharedAudioContext) {
+                this._sharedAudioContext = new AudioContext();
+            }
+            const ctx = this._sharedAudioContext;
+            const source = ctx.createMediaStreamSource(stream);
+            const analyser = ctx.createAnalyser();
             analyser.fftSize = 256;
             source.connect(analyser);
             const data = new Uint8Array(analyser.frequencyBinCount);
-            this._analysers.set(userId, { analyser, data, context: audioCtx });
+            this._analysers.set(userId, { source, analyser, data });
         }
         catch (err) {
             if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err);
@@ -469,12 +497,20 @@ export class Voip extends Behaviour {
     private cleanupAnalyser(userId: string) {
         const info = this._analysers.get(userId);
         if (info) {
-            info.context.close();
+            info.source.disconnect();
+            info.analyser.disconnect();
             this._analysers.delete(userId);
         }
         this._speakingStates.delete(userId);
     }
 
+    private closeSharedAudioContext() {
+        if (this._sharedAudioContext) {
+            this._sharedAudioContext.close();
+            this._sharedAudioContext = undefined;
+        }
+    }
+
     private onReceiveStream = (evt: StreamReceivedEvent) => {
         const userId = evt.target.userId;
         const stream = evt.stream;
@@ -512,12 +548,11 @@ export class Voip extends Behaviour {
 
     private onVisibilityChanged = () => {
         if (this.runInBackground) return;
-        const visible = document.visibilityState === "visible";
-        const muted = !visible;
+        const muted = document.visibilityState !== "visible";
+        // Mute incoming so we don't hear other users while tab is hidden.
         this.setMuted(muted);
-        for (const element of this._incomingStreams) {
-            const str = element[1];
-            str.muted = muted;
-        }
+        // Also disable our outgoing mic tracks (cheaper than disconnect/reconnect — keeps the mic permission).
+        const tracks = this._outputStream?.getAudioTracks();
+        if (tracks) for (const t of tracks) t.enabled = !muted;
     };
 }

package/src/engine-components/api.ts

@@ -41,6 +41,7 @@ export { Collider } from "./Collider.js"; // export abstract type
 export { Behaviour, Component, GameObject } from "./Component.js";
 
 // We dont want to export everything in the extensions
+export { AudioRolloffMode } from "./AudioSource.js";
 export { ClearFlags } from "./Camera.js"
 export { DragMode } from "./DragControls.js";
 export type { DropListenerNetworkEventArguments, DropListenerOnDropArguments } from "./DropListener.js";

package/src/engine-components/export/usdz/USDZExporter.ts

@@ -3,7 +3,7 @@ import { Euler, Material, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "t
 
 import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
 import { findObjectOfType } from "../../../engine/engine_components.js";
-import { _cxKhKwDL } from "../../../engine/engine_license.js";
+import { UsFaeEU } from "../../../engine/engine_license.js";
 import { serializable } from "../../../engine/engine_serialization.js";
 import { getFormattedDate, Progress } from "../../../engine/engine_time_utils.js";
 import { DeviceUtilities, getParam } from "../../../engine/engine_utils.js";
@@ -277,7 +277,7 @@ export class USDZExporter extends Behaviour {
         let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
         name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
 
-        if (!_cxKhKwDL()) {
+        if (!UsFaeEU()) {
             if (name !== "") name += "-";
             name += "MadeWithNeedle";
         }
@@ -682,7 +682,7 @@ export class USDZExporter extends Behaviour {
                     if (debug)
                         showBalloonMessage("Quicklook url: " + callToActionURL);
                     if (callToActionURL) {
-                        if (!_cxKhKwDL()) {
+                        if (!UsFaeEU()) {
                             console.warn("Quicklook closed: custom redirects require a Needle Engine Pro license: https://needle.tools/pricing", callToActionURL)
                         }
                         else {
@@ -697,7 +697,7 @@ export class USDZExporter extends Behaviour {
     private buildQuicklookOverlay(): CustomBranding {
         const obj: CustomBranding = {};
         if (this.customBranding) Object.assign(obj, this.customBranding);
-        if (!_cxKhKwDL()) {
+        if (!UsFaeEU()) {
             console.log("Custom Quicklook banner text requires pro license: https://needle.tools/pricing");
             obj.callToAction = "Close";
             obj.checkoutTitle = "🌵 Made with Needle";