@needle-tools/engine 5.0.3 → 5.1.0-alpha.1

package/src/engine-components/Voip.ts

@@ -3,10 +3,10 @@ import { Application } from "../engine/engine_application.js";
 import { RoomEvents } from "../engine/engine_networking.js";
 import { disposeStream, NetworkedStreamEvents, NetworkedStreams, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js"
 import { serializable } from "../engine/engine_serialization_decorator.js";
-import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
-import { delay } from "../engine/engine_utils.js";
+import { delay, DeviceUtilities, getParam } from "../engine/engine_utils.js";
 import { getIconElement } from "../engine/webcomponents/icons.js";
 import { Behaviour } from "./Component.js";
+import { EventList } from "./EventList.js";
 
 export const noVoip = "noVoip";
 const debugParam = getParam("debugvoip");
@@ -75,6 +75,66 @@ export class Voip extends Behaviour {
      */
     debug: boolean = false;
 
+    private _volume: number = 1;
+
+    /**
+     * Volume for incoming audio streams (0 = silent, 1 = full volume).
+     * Changes apply immediately to all active incoming streams.
+     */
+    @serializable()
+    get volume(): number { return this._volume; }
+    set volume(val: number) {
+        this._volume = val;
+        for (const audio of this._incomingStreams.values()) {
+            audio.volume = val;
+        }
+    }
+
+    /**
+     * Get the incoming audio element for a specific user.
+     * Use this to route audio through the Web Audio API for spatial audio, effects, or analysis.
+     * @param userId The connection ID of the remote user
+     * @returns The HTMLAudioElement for this user's stream, or undefined if not connected
+     * @example
+     * ```ts
+     * const audioEl = voip.getAudioElement(userId);
+     * if (audioEl) {
+     *   const audioCtx = new AudioContext();
+     *   const source = audioCtx.createMediaElementSource(audioEl);
+     *   const panner = audioCtx.createPanner();
+     *   source.connect(panner).connect(audioCtx.destination);
+     * }
+     * ```
+     */
+    getAudioElement(userId: string): HTMLAudioElement | undefined {
+        return this._incomingStreams.get(userId);
+    }
+
+    /**
+     * Get all active incoming audio streams as a Map of userId → HTMLAudioElement.
+     * Useful for iterating over all connected voice users.
+     */
+    get incomingStreams(): ReadonlyMap<string, HTMLAudioElement> {
+        return this._incomingStreams;
+    }
+
+    /**
+     * Threshold for speaking detection (0–255). When a user's audio amplitude exceeds this,
+     * they are considered "speaking". Default is 30.
+     */
+    @serializable()
+    speakingThreshold: number = 30;
+
+    /**
+     * Event fired when a user's speaking state changes.
+     * Passes `{ userId: string, isSpeaking: boolean, volume: number }`.
+     */
+    @serializable(EventList)
+    onSpeakingChanged: EventList = new EventList();
+
+    private _speakingStates = new Map<string, boolean>();
+    private _analysers = new Map<string, { analyser: AnalyserNode, data: Uint8Array, context: AudioContext }>();
+
     private _net?: NetworkedStreams;
     private _menubutton?: HTMLElement;
 
@@ -141,12 +201,25 @@ export class Voip extends Behaviour {
         this.onEnabledChanged();
         this.updateButton();
         window.removeEventListener("visibilitychange", this.onVisibilityChanged);
+        // Clean up analysers
+        for (const userId of [...this._analysers.keys()]) {
+            this.cleanupAnalyser(userId);
+        }
     }
 
     /** @internal */
     onDestroy(): void {
         this._menubutton?.remove();
         this._menubutton = undefined;
+        // Clean up all streams and analysers
+        for (const userId of [...this._analysers.keys()]) {
+            this.cleanupAnalyser(userId);
+        }
+        for (const incoming of this._incomingStreams.values()) {
+            disposeStream(incoming.srcObject as MediaStream);
+        }
+        this._incomingStreams.clear();
+        this._speakingStates.clear();
     }
 
     /** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */
@@ -347,10 +420,61 @@ export class Voip extends Behaviour {
             disposeStream(incoming.srcObject as MediaStream);
         }
         this._incomingStreams.clear();
+        for (const userId of this._analysers.keys()) {
+            this.cleanupAnalyser(userId);
+        }
     }
 
     private _incomingStreams: Map<string, HTMLAudioElement> = new Map();
 
+    /** @internal */
+    update() {
+        // Only run speaking detection if someone is listening
+        if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return;
+
+        for (const [userId, info] of this._analysers) {
+            info.analyser.getByteFrequencyData(info.data as Uint8Array<ArrayBuffer>);
+            // Average amplitude
+            let sum = 0;
+            for (let i = 0; i < info.data.length; i++) sum += info.data[i];
+            const avg = sum / info.data.length;
+
+            const wasSpeaking = this._speakingStates.get(userId) ?? false;
+            const isSpeaking = avg > this.speakingThreshold;
+
+            if (isSpeaking !== wasSpeaking) {
+                this._speakingStates.set(userId, isSpeaking);
+                this.onSpeakingChanged.invoke({ userId, isSpeaking, volume: avg / 255 });
+            }
+        }
+    }
+
+    private setupAnalyser(userId: string, _audioElement: HTMLAudioElement, stream: MediaStream) {
+        // Only set up if someone is listening or might listen
+        if (this._analysers.has(userId)) return;
+        try {
+            const audioCtx = new AudioContext();
+            const source = audioCtx.createMediaStreamSource(stream);
+            const analyser = audioCtx.createAnalyser();
+            analyser.fftSize = 256;
+            source.connect(analyser);
+            const data = new Uint8Array(analyser.frequencyBinCount);
+            this._analysers.set(userId, { analyser, data, context: audioCtx });
+        }
+        catch (err) {
+            if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err);
+        }
+    }
+
+    private cleanupAnalyser(userId: string) {
+        const info = this._analysers.get(userId);
+        if (info) {
+            info.context.close();
+            this._analysers.delete(userId);
+        }
+        this._speakingStates.delete(userId);
+    }
+
     private onReceiveStream = (evt: StreamReceivedEvent) => {
         const userId = evt.target.userId;
         const stream = evt.stream;
@@ -361,7 +485,9 @@ export class Voip extends Behaviour {
             this._incomingStreams.set(userId, audioElement);
         }
         audioElement.srcObject = stream;
+        audioElement.volume = this._volume;
         audioElement.setAttribute("autoplay", "true");
+        this.setupAnalyser(userId, audioElement, stream);
         // for mobile we need to wait for user interaction to play audio. Auto play doesnt work on android when the page is refreshed
         Application.registerWaitForInteraction(() => {
             audioElement?.play().catch((err) => {
@@ -374,6 +500,7 @@ export class Voip extends Behaviour {
         const existing = this._incomingStreams.get(evt.userId);
         disposeStream(existing?.srcObject as MediaStream);
         this._incomingStreams.delete(evt.userId);
+        this.cleanupAnalyser(evt.userId);
     }
 
     private onEnabledChanged = () => {

package/src/engine-components/api.ts

@@ -35,6 +35,7 @@
  */
 
 export * from "./codegen/components.js";
+export { AnimatorControllerBuilder, type ConditionMode, type StateOptions, type TransitionOptions } from "./AnimatorController.js";
 export { Collider } from "./Collider.js"; // export abstract type
 export { Behaviour, Component, GameObject } from "./Component.js";
 

package/src/engine-components/codegen/components.ts

@@ -6,6 +6,7 @@ export { Animation } from "../Animation.js";
 export { Keyframe } from "../AnimationCurve.js";
 export { AnimationCurve } from "../AnimationCurve.js";
 export { Animator } from "../Animator.js";
+export { AnimatorControllerBuilder } from "../AnimatorController.js";
 export { AnimatorController } from "../AnimatorController.js";
 export { AudioListener } from "../AudioListener.js";
 export { AudioSource } from "../AudioSource.js";

package/src/engine-components/postprocessing/Effects/Tonemapping.ts

@@ -1,11 +1,11 @@
 import type { ToneMappingEffect as _TonemappingEffect, ToneMappingMode } from "postprocessing";
+import type { ToneMapping } from "three";
 
 import { MODULES } from "../../../engine/engine_modules.js";
 import { serializable } from "../../../engine/engine_serialization.js";
 import { nameToThreeTonemapping } from "../../../engine/engine_tonemapping.js";
 import { getParam } from "../../../engine/engine_utils.js";
 import { EffectProviderResult, PostProcessingEffect } from "../PostProcessingEffect.js";
-import { findPostProcessingManager } from "../utils.js";
 import { VolumeParameter } from "../VolumeParameter.js";
 import { registerCustomEffectType } from "../VolumeProfile.js";
 import { NEToneMappingMode, NEToneMappingModeNames, threeToNeedleToneMapping, threeToneMappingToEffectMode, toThreeToneMapping } from "./Tonemapping.utils.js";
@@ -14,8 +14,8 @@ const debug = getParam("debugpost");
 
 
 /**
- * [ToneMappingEffect](https://engine.needle.tools/docs/api/ToneMappingEffect) adjusts the brightness and contrast of the rendered scene to map high dynamic range (HDR) colors to a displayable range.  
- * This effect is essential for achieving realistic lighting and color representation in 3D scenes, as it helps to preserve details in both bright and dark areas.  
+ * [ToneMappingEffect](https://engine.needle.tools/docs/api/ToneMappingEffect) adjusts the brightness and contrast of the rendered scene to map high dynamic range (HDR) colors to a displayable range.
+ * This effect is essential for achieving realistic lighting and color representation in 3D scenes, as it helps to preserve details in both bright and dark areas.
  * Various tonemapping algorithms can be applied to achieve different visual styles and effects.
  * @summary Tonemapping Post-Processing Effect
  * @category Effects
@@ -46,18 +46,23 @@ export class ToneMappingEffect extends PostProcessingEffect {
 
     get isToneMapping() { return true; }
 
-    onEffectEnabled(): void {
-        // Tonemapping works with and without a postprocessing manager. 
-        // If there's no manager already in the scene we don't need to create one because tonemapping can also be applied without a postprocessing pass
-        const ppmanager = findPostProcessingManager(this);
-        if (!ppmanager) return;
-        super.onEffectEnabled(ppmanager);
+    /** The three.js ToneMapping enum value resolved from the current mode */
+    get threeToneMapping(): ToneMapping {
+        if (this.mode.isInitialized && this.mode.overrideState)
+            return toThreeToneMapping(this.mode.value);
+        return this.context.renderer.toneMapping as ToneMapping;
+    }
+
+    /** The exposure value to apply */
+    get toneMappingExposure(): number {
+        if (this.exposure.overrideState && this.exposure.value !== undefined)
+            return Math.max(0.0, this.exposure.value);
+        return this.context.renderer.toneMappingExposure;
     }
 
     private _tonemappingEffect: _TonemappingEffect | null = null;
     onCreateEffect(): EffectProviderResult | undefined {
 
-
         // ensure the effect tonemapping value is initialized
         if (this.mode.isInitialized == false) {
             const mode = threeToNeedleToneMapping(this.context.renderer.toneMapping);
@@ -90,19 +95,6 @@ export class ToneMappingEffect extends PostProcessingEffect {
         return tonemapping;
     }
 
-
-    onBeforeRender(): void {
-        if (this._tonemappingEffect && this.postprocessingContext?.handler.getEffectIsActive(this._tonemappingEffect)) {
-            if (this.mode.overrideState)
-                this.context.renderer.toneMapping = toThreeToneMapping(this.mode.value);
-            if (this.exposure.overrideState && this.exposure.value !== undefined) {
-                const newValue = Math.max(0.0, this.exposure.value);
-                this.context.renderer.toneMappingExposure = newValue;
-            }
-        }
-    }
-
-
 }
 
-registerCustomEffectType("Tonemapping", ToneMappingEffect);
+registerCustomEffectType("Tonemapping", ToneMappingEffect);

package/src/engine-components/postprocessing/PostProcessingEffect.ts

@@ -3,9 +3,10 @@ import type { Effect, Pass } from "postprocessing";
 import type { EditorModification, IEditorModification } from "../../engine/engine_editor-sync.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { getParam } from "../../engine/engine_utils.js";
+import type { PostProcessing } from "../../engine/postprocessing/index.js";
+import type { IPostProcessingEffect } from "../../engine/postprocessing/types.js";
 import { Component } from "../Component.js";
 import type { PostProcessingHandler } from "./PostProcessingHandler.js";
-import { getPostProcessingManager, IPostProcessingManager } from "./utils.js";
 import { VolumeParameter } from "./VolumeParameter.js";
 
 const debug = getParam("debugpost");
@@ -50,8 +51,9 @@ export interface IEffectProvider {
  * 
  * @category Effects
  * @group Components
+ * @see {@link PostProcessing} for core Needle Engine postprocessing control, also accessible via `context.postprocessing`
  */
-export abstract class PostProcessingEffect extends Component implements IEffectProvider, IEditorModification {
+export abstract class PostProcessingEffect extends Component implements IPostProcessingEffect, IEffectProvider, IEditorModification {
 
     get isPostProcessingEffect() { return true; }
 
@@ -101,31 +103,22 @@ export abstract class PostProcessingEffect extends Component implements IEffectP
     @serializable()
     active: boolean = true;
 
-    private _manager: IPostProcessingManager | null = null;
-
     onEnable(): void {
         super.onEnable();
         if (debug) console.warn("Enable", this.constructor.name + (!this.__internalDidAwakeAndStart ? " (awake)" : ""));
-        // Dont override the serialized value by enabling (we could also just disable this component / map enabled to active)
-        if (this.__internalDidAwakeAndStart)
-            this.active = true;
-        this.onEffectEnabled();
+        // Ensure active is synced with the component's enabled state
+        this.active = true;
+        // Register directly with the core postprocessing stack
+        this.context.postprocessing.addEffect(this);
     }
 
     onDisable(): void {
         super.onDisable();
         if (debug) console.warn("Disable", this.constructor.name);
-        this._manager?.removeEffect(this);
+        this.context.postprocessing.removeEffect(this);
         this.active = false;
     }
 
-    protected onEffectEnabled(manager?: IPostProcessingManager) {
-        if (manager && manager.isPostProcessingManager === true) this._manager = manager;
-        else if (!this._manager) this._manager = getPostProcessingManager(this);
-        this._manager!.addEffect(this);
-        this._manager!.dirty = true;
-    }
-
     /** override to initialize bindings on parameters */
     init() { }
 

package/src/engine-components/postprocessing/PostProcessingHandler.ts

@@ -8,6 +8,7 @@ import { Context } from "../../engine/engine_setup.js";
 import { Graphics } from "../../engine/engine_three_utils.js";
 import { Constructor } from "../../engine/engine_types.js";
 import { getParam } from "../../engine/engine_utils.js";
+import type { IPostProcessingHandler } from "../../engine/postprocessing/types.js";
 import { Camera } from "../Camera.js";
 import { threeToneMappingToEffectMode } from "./Effects/Tonemapping.utils.js";
 import { PostProcessingEffect, PostProcessingEffectContext } from "./PostProcessingEffect.js";
@@ -26,7 +27,7 @@ const previousToneMapping = Symbol("needle:previous-tone-mapping");
 /**
  * [PostProcessingHandler](https://engine.needle.tools/docs/api/PostProcessingHandler) Is responsible for applying post processing effects to the scene. It is internally used by the {@link Volume} component
  */
-export class PostProcessingHandler {
+export class PostProcessingHandler implements IPostProcessingHandler {
 
     private _composer: EffectComposer | null = null;
     private _lastVolumeComponents?: PostProcessingEffect[];