@needle-tools/engine 5.0.4 → 5.0.5

package/src/engine/engine_audio.ts

@@ -21,4 +21,188 @@ export function ensureAudioContextIsResumed() {
             }, 500);
         });
     });
+}
+
+
+/**
+ * Represents an audio clip that can be loaded and played independently.
+ * The AudioClip class encapsulates the URL of the audio resource and provides
+ * methods for playback control (play, pause, stop) and querying duration.
+ */
+export class AudioClip {
+
+    /**
+     * Creates a new AudioClip instance with the specified URL.
+     * @param url The URL of the audio resource to load. This can be a path to an audio file or a MediaStream URL.
+     */
+    constructor(public readonly url: string) {
+    }
+
+    /** Whether the clip is currently playing.
+     * @returns `true` if the clip is actively playing audio.
+     */
+    get isPlaying(): boolean {
+        return this._audioElement !== undefined
+            && !this._audioElement.paused
+            && !this._audioElement.ended;
+    }
+
+    /**
+     * The total duration of the audio clip in seconds.
+     * Loads the audio metadata if not already available.
+     * @returns A promise that resolves with the duration in seconds.
+     */
+    getDuration(): Promise<number> {
+        if (this._duration !== undefined) {
+            return Promise.resolve(this._duration);
+        }
+        return this.ensureAudioElement().then(audio => {
+            this._duration = audio.duration;
+            return audio.duration;
+        });
+    }
+
+    /**
+     * Plays the audio clip from the current position.
+     * @returns A promise that resolves when playback finishes, or rejects on error.
+     * If the clip is looping, the promise will never resolve on its own – call {@link stop} or {@link pause} to end playback.
+     */
+    // #region Play
+    play(): Promise<void> {
+        return this.ensureAudioElement().then(audio => {
+            return new Promise<void>((resolve, reject) => {
+                const onEnded = () => {
+                    cleanup();
+                    resolve();
+                };
+                const onError = () => {
+                    cleanup();
+                    reject(new Error(`Playback error for ${this.url}`));
+                };
+                const onPause = () => {
+                    // pause/stop also resolve the promise
+                    cleanup();
+                    resolve();
+                };
+                const cleanup = () => {
+                    audio.removeEventListener("ended", onEnded);
+                    audio.removeEventListener("error", onError);
+                    audio.removeEventListener("pause", onPause);
+                };
+                audio.addEventListener("ended", onEnded);
+                audio.addEventListener("error", onError);
+                audio.addEventListener("pause", onPause);
+                audio.play().catch(err => {
+                    cleanup();
+                    reject(err);
+                });
+            });
+        });
+    }
+
+    /**
+     * Pauses playback at the current position.
+     * Call {@link play} to resume.
+     */
+    // #region Pause/Stop
+    pause(): void {
+        this._audioElement?.pause();
+    }
+
+    /**
+     * Stops playback and resets the position to the beginning.
+     */
+    stop(): void {
+        if (this._audioElement) {
+            this._audioElement.pause();
+            this._audioElement.currentTime = 0;
+        }
+    }
+
+    /** Whether the clip should loop when reaching the end. */
+    get loop(): boolean { return this._loop; }
+    set loop(value: boolean) {
+        this._loop = value;
+        if (this._audioElement) this._audioElement.loop = value;
+    }
+
+    /** Playback volume from 0 (silent) to 1 (full). */
+    get volume(): number { return this._volume; }
+    set volume(value: number) {
+        this._volume = value;
+        if (this._audioElement) this._audioElement.volume = value;
+    }
+
+    /** Current playback position in seconds. */
+    get currentTime(): number { return this._audioElement?.currentTime ?? 0; }
+    set currentTime(value: number) {
+        if (this._audioElement) this._audioElement.currentTime = value;
+    }
+
+    /** Normalized playback progress from 0 to 1.
+     * @returns The current playback position as a value between 0 and 1, or 0 if the duration is unknown.
+     */
+    get progress(): number {
+        if (!this._audioElement || !this._duration) return 0;
+        return this._audioElement.currentTime / this._duration;
+    }
+
+    /**
+     * Seeks to a normalized position (0–1) in the clip.
+     * @param position A value between 0 (start) and 1 (end).
+     */
+    // #region Seek
+    seek(position: number): void {
+        if (this._audioElement && this._duration) {
+            this._audioElement.currentTime = Math.max(0, Math.min(1, position)) * this._duration;
+        }
+    }
+
+    /** The underlying HTMLAudioElement, or `undefined` if not yet created.
+     * Use this to connect the element to the Web Audio API via `createMediaElementSource()`.
+     * @returns The HTMLAudioElement if the clip has been loaded or played, otherwise `undefined`.
+     */
+    get audioElement(): HTMLAudioElement | undefined { return this._audioElement; }
+
+    private _audioElement?: HTMLAudioElement;
+    private _duration?: number;
+    private _loadPromise?: Promise<HTMLAudioElement>;
+    private _loop: boolean = false;
+    private _volume: number = 1;
+
+    /** Lazily creates and loads the shared HTMLAudioElement. */
+    private ensureAudioElement(): Promise<HTMLAudioElement> {
+        if (this._audioElement && this._loadPromise) {
+            return this._loadPromise;
+        }
+        const audio = this._audioElement ?? new Audio(this.url);
+        this._audioElement = audio;
+        audio.loop = this._loop;
+        audio.volume = this._volume;
+
+        if (audio.readyState >= HTMLMediaElement.HAVE_METADATA) {
+            this._duration = audio.duration;
+            this._loadPromise = Promise.resolve(audio);
+            return this._loadPromise;
+        }
+
+        this._loadPromise = new Promise<HTMLAudioElement>((resolve, reject) => {
+            const onLoaded = () => {
+                cleanup();
+                this._duration = audio.duration;
+                resolve(audio);
+            };
+            const onError = (e: Event) => {
+                cleanup();
+                reject(new Error(`Failed to load audio clip from ${this.url}: ${e}`));
+            };
+            const cleanup = () => {
+                audio.removeEventListener("loadedmetadata", onLoaded);
+                audio.removeEventListener("error", onError);
+            };
+            audio.addEventListener("loadedmetadata", onLoaded);
+            audio.addEventListener("error", onError);
+        });
+        return this._loadPromise;
+    }
 }

package/src/engine/engine_gameobject.ts

@@ -207,8 +207,8 @@ function internalDestroy(instance: Object3D | Component, recursive: boolean = tr
         if (comp[$isDontDestroy]) return;
         destroyed_components.push(comp);
         const go = comp.gameObject;
-        comp.__internalDisable();
-        comp.__internalDestroy();
+        comp.__internalDisable?.();
+        comp.__internalDestroy?.();
         comp.gameObject = go;
         return;
     }

package/src/engine/engine_init.ts

@@ -1,7 +1,10 @@
+import { initAnimatorControllerSerializer } from "../engine-components/AnimatorController.js";
 import { initAnimationAutoplay } from "../engine-components/AnimationUtilsAutoplay.js";
 import { initCameraUtils } from "../engine-components/CameraUtils.js";
+import { initVolumeParameterSerializer } from "../engine-components/postprocessing/VolumeParameter.js";
 import { initSceneSwitcherAttributes } from "../engine-components/SceneSwitcher.js";
 import { initSkyboxAttributes } from "../engine-components/Skybox.js";
+import { initAddressableSerializers } from "./engine_addressables.js";
 import { initBuiltinTypes } from "./codegen/register_types.js";
 import { ensureAudioContextIsResumed } from "./engine_audio.js";
 import { initNeedleLoader } from "./engine_loaders.js";
@@ -13,6 +16,7 @@ import { patchLayers } from "./js-extensions/Layers.js";
 import { initObject3DExtensions } from "./js-extensions/Object3D.js";
 import { initVectorExtensions } from "./js-extensions/Vector.js";
 import { initWebComponents } from "./webcomponents/init.js";
+import { initSpatialConsole } from "./debug/debug_spatial_console.js";
 
 let initialized = false;
 
@@ -32,6 +36,9 @@ export function initEngine() {
     initWebComponents();
     initShims();
     initBuiltinSerializers();
+    initAddressableSerializers();
+    initAnimatorControllerSerializer();
+    initVolumeParameterSerializer();
     patchTonemapping();
     patchLayers();
     initCameraExtensions();
@@ -44,4 +51,5 @@ export function initEngine() {
     initAnimationAutoplay();
     initSkyboxAttributes();
     initSceneSwitcherAttributes();
+    initSpatialConsole();
 }

package/src/engine/engine_mainloop_utils.ts

@@ -344,8 +344,11 @@ function updateIsActiveInHierarchyRecursiveRuntime(go: Object3D, activeInHierarc
         if (allowEventCall) {
             const components = go.userData?.components;
             if (components) {
-                for (let ci = components.length - 1, cl = -1; ci > cl; ci--) {
-                    const comp = components[ci];
+                // We need to iterate on components in the original order right now to work-around UI related initialization bugs where RectTransform must be initialized before e.g. Graphic components. https://linear.app/needle/issue/NE-6986
+                // In the future we can reverse iterate this once the UI system has been replaced (that being said it's probably expected that component enable in the order in which they're added to an object)
+                const componentsCopy = [...components];
+                for (let ci = 0; ci < componentsCopy.length; ci++) {
+                    const comp = componentsCopy[ci];
                     if (activeInHierarchy) {
                         if (comp?.enabled) {
                             try { comp.__internalAwake(); }

package/src/engine/engine_serialization_builtin_serializer.ts

@@ -4,6 +4,7 @@ import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../eng
 import { Behaviour, Component, GameObject } from "../engine-components/Component.js";
 import { CallInfo, EventList } from "../engine-components/EventList.js";
 import { AssetReference } from "./engine_addressables.js";
+import { AudioClip } from "./engine_audio.js";
 import { debugExtension } from "./engine_default_parameters.js";
 import { SerializationContext, TypeSerializer } from "./engine_serialization_core.js";
 import { RenderTexture } from "./engine_texture.js";
@@ -11,7 +12,7 @@ import { IComponent } from "./engine_types.js";
 import { resolveUrl } from "./engine_utils.js";
 import { RGBAColor } from "./js-extensions/index.js";
 
-
+// #region Color
 class ColorSerializer extends TypeSerializer {
     constructor() {
         super([Color, RGBAColor], "ColorSerializer")
@@ -35,6 +36,8 @@ class ColorSerializer extends TypeSerializer {
     }
 }
 
+// #region Euler
+
 class EulerSerializer extends TypeSerializer {
     constructor() {
         super([Euler], "EulerSerializer");
@@ -58,6 +61,8 @@ declare type ObjectData = {
     node?: number;
     guid?: string;
 }
+
+// #region ObjectSerializer
 class ObjectSerializer extends TypeSerializer {
     constructor() {
         super(Object3D, "ObjectSerializer");
@@ -137,7 +142,7 @@ class ObjectSerializer extends TypeSerializer {
     }
 }
 
-
+// #region ComponentSerializer
 class ComponentSerializer extends TypeSerializer {
 
     constructor() {
@@ -226,6 +231,7 @@ declare type EventListCall = {
 
 const $eventListDebugInfo = Symbol("eventListDebugInfo");
 
+// #region EventListSerializer
 class EventListSerializer extends TypeSerializer {
     constructor() {
         super([EventList]);
@@ -371,6 +377,7 @@ class EventListSerializer extends TypeSerializer {
  */
 const cloneOriginalMap = new WeakMap<Texture, Texture>();
 
+// #region RenderTextureSerializer
 export class RenderTextureSerializer extends TypeSerializer {
     constructor() {
         super([RenderTexture, WebGLRenderTarget]);
@@ -412,7 +419,7 @@ export class RenderTextureSerializer extends TypeSerializer {
     }
 }
 
-
+// #region UriSerializer
 export class UriSerializer extends TypeSerializer {
     constructor() {
         super([URL]);
@@ -430,7 +437,27 @@ export class UriSerializer extends TypeSerializer {
     }
 }
 
+// #region AudioClipSerializer
+class AudioClipSerializer extends TypeSerializer {
+    constructor() {
+        super([AudioClip]);
+    }
+
+    onSerialize(_data: AudioClip, _context: SerializationContext) {
+        return null;
+    }
+
+    onDeserialize(data: string, context: SerializationContext) {
+        if (typeof data === "string" && data.length > 0) {
+            const url = resolveUrl(context.gltfId, data);
+            if (url) return new AudioClip(url);
+        }
+        return undefined;
+    }
+}
+
 
+// #region Init serializer
 // Module-level references used by EventListSerializer internally
 export let colorSerializer: ColorSerializer;
 export let objectSerializer: ObjectSerializer;
@@ -460,4 +487,5 @@ export function initBuiltinSerializers() {
     eventListSerializer = new EventListSerializer();
     new RenderTextureSerializer();
     new UriSerializer();
+    new AudioClipSerializer();
 }

package/src/engine/webcomponents/needle-engine.ts

@@ -40,11 +40,17 @@ export interface NeedleEngineAttributes {
     'hash': string;
     /** Set to automatically add OrbitControls to the loaded scene. */
     'camera-controls': string;
-    /** Override the default draco decoder path location. */
+    /** Override the default Draco decoder/decompressor path. Can be a URL or a local path to a directory containing the Draco decoder files.
+     * @default "https://www.gstatic.com/draco/versioned/decoders/1.5.7/"
+     * @example <needle-engine dracoDecoderPath="./decoders/draco/"></needle-engine>
+     */
     'dracoDecoderPath': string;
-    /** Override the default draco library type. */
+    /** Override the default Draco decoder type. */
     'dracoDecoderType': 'wasm' | 'js';
-    /** Override the default KTX2 transcoder/decoder path. */
+    /** Override the default KTX2 transcoder/decoder path. Can be a URL or a local path to a directory containing the KTX2 transcoder files.
+     * @default "https://cdn.needle.tools/static/three/0.179.1/basis2/"
+     * @example <needle-engine ktx2DecoderPath="./decoders/ktx2/"></needle-engine>
+     */
     'ktx2DecoderPath': string;
     /** Prevent context from being disposed when element is removed from DOM. */
     'keep-alive': 'true' | 'false';

package/src/engine/xr/NeedleXRSession.ts

@@ -805,15 +805,16 @@ export class NeedleXRSession implements INeedleXRSession {
     get viewerPose(): XRViewerPose | undefined { return this._viewerPose; }
 
 
-    /** @returns `true` if any image is currently being tracked */
-    /** returns true if images are currently being tracked */
+    /** @returns `true` if any image is currently being tracked or emulated */
     get isTrackingImages() {
         if (this.frame && "getImageTrackingResults" in this.frame && typeof this.frame.getImageTrackingResults === "function") {
             try {
                 const trackingResult = this.frame.getImageTrackingResults();
                 for (const result of trackingResult) {
                     const state = result.trackingState;
-                    if (state === "tracked") return true;
+                    if (state === "tracked" || state === "emulated") {
+                        return true;
+                    }
                 }
             }
             catch {
@@ -1018,6 +1019,8 @@ export class NeedleXRSession implements INeedleXRSession {
     private readonly _xr_update_scripts: INeedleXRSessionEventReceiver[] = [];
     /** scripts that are in the scene but inactive (e.g. disabled parent gameObject) */
     private readonly _inactive_scripts: INeedleXRSessionEventReceiver[] = [];
+    /** tracks scripts that have received onEnterXR — prevents spurious onLeaveXR calls */
+    private readonly _scripts_in_xr = new Set<INeedleXRSessionEventReceiver>();
     private readonly _controllerAdded: ControllerChangedEvt[];
     private readonly _controllerRemoved: ControllerChangedEvt[];
     private readonly _originalCameraWorldPosition?: Vector3 | null;
@@ -1128,7 +1131,18 @@ export class NeedleXRSession implements INeedleXRSession {
         // we set the session on the webxr manager at the end because we want to receive inputsource events first
         // e.g. in case there's a bug in the threejs codebase
         this.context.xr = this;
-        this.context.renderer.xr.setSession(this.session).then(this.onRendererSessionSet);
+        this.context.renderer.xr.setSession(this.session).then(this.onRendererSessionSet)
+            .catch(err => {
+                // Workaround for WebXR emulators on localhost: the polyfilled XRSession fails the
+                // native XRWebGLBinding constructor's instanceof check. Retry with layers disabled.
+                // See https://github.com/meta-quest/immersive-web-emulator/issues/80
+                if (isDevEnvironment() && err instanceof TypeError && typeof XRWebGLBinding !== "undefined") {
+                    console.error(`XRWebGLBinding failed (${err.message}).\nSee https://github.com/meta-quest/immersive-web-emulator/issues/80`);
+                }
+                else {
+                    console.error("renderer.xr.setSession failed:", err);
+                }
+            });
         // disable three.js renderer controller autoUpdate (added in ac67b31e3548386f8a93e23a4176554c92bbd0d9)
         if ("controllerAutoUpdate" in this.context.renderer.xr) {
             console.debug("Disabling three.js controllerAutoUpdate");
@@ -1296,8 +1310,10 @@ export class NeedleXRSession implements INeedleXRSession {
         // even if they might already be destroyed e.g. by the WebXR component (it destroys the default controller scripts)
         // they should still receive this callback to be properly cleaned up
         for (const listener of this._xr_scripts) {
+            this._scripts_in_xr.delete(listener);
             listener?.onLeaveXR?.({ xr: this });
         }
+        this._scripts_in_xr.clear();
 
         this.sync?.onExitXR(this);
 
@@ -1364,10 +1380,17 @@ export class NeedleXRSession implements INeedleXRSession {
         }
 
         // make sure the camera is parented to the active rig
-        if (this.rig && this._mainCamera?.gameObject) {
-            const currentParent = this._mainCamera?.gameObject?.parent;
-            if (currentParent !== this.rig.gameObject) {
-                this.rig.gameObject.add(this._mainCamera?.gameObject);
+        // Note: applyCustomForward() inserts _cameraRenderParent between rig and camera,
+        // so the hierarchy is: rig → _cameraRenderParent → camera. We check the effective parent.
+        if (this.rig) {
+            const cameraObject = this._mainCamera?.gameObject ?? this.context.mainCamera;
+            if (cameraObject) {
+                const effectiveParent = cameraObject.parent === this._cameraRenderParent
+                    ? this._cameraRenderParent.parent
+                    : cameraObject.parent;
+                if (effectiveParent !== this.rig.gameObject) {
+                    this.rig.gameObject.add(cameraObject);
+                }
             }
         }
 
@@ -1499,8 +1522,12 @@ export class NeedleXRSession implements INeedleXRSession {
         }
 
         //performance.mark('NeedleXRSession update scripts start');
-        // invoke update on all scripts
-        for (const script of this._xr_update_scripts) {
+        // check all XR scripts for destroyed or inactive state
+        // this must cover _xr_scripts (not just _xr_update_scripts) so that scripts
+        // without onUpdateXR are also detected as inactive when removed from the scene
+        // iterate backwards since markInactive/removeScript modifies the array
+        for (let i = this._xr_scripts.length - 1; i >= 0; i--) {
+            const script = this._xr_scripts[i];
             if (script.destroyed === true) {
                 this._script_to_remove.push(script);
                 continue;
@@ -1509,6 +1536,10 @@ export class NeedleXRSession implements INeedleXRSession {
                 this.markInactive(script);
                 continue;
             }
+        }
+        // invoke update on scripts that have onUpdateXR
+        for (const script of this._xr_update_scripts) {
+            if (script.destroyed || script.activeAndEnabled === false) continue;
             if (script.onUpdateXR) script.onUpdateXR(args);
         }
         //performance.mark('NeedleXRSession update scripts end');
@@ -1631,9 +1662,11 @@ export class NeedleXRSession implements INeedleXRSession {
                 const script = this._inactive_scripts[i];
                 if (script.activeAndEnabled) {
                     this._inactive_scripts.splice(i, 1);
-                    this.addScript(script);
-                    this.invokeCallback_EnterXR(script);
-                    for (const ctrl of this.controllers) this.invokeCallback_ControllerAdded(script, ctrl);
+                    // addScript returns false if already re-added (e.g. via new_scripts_xr processing)
+                    if (this.addScript(script)) {
+                        this.invokeCallback_EnterXR(script);
+                        for (const ctrl of this.controllers) this.invokeCallback_ControllerAdded(script, ctrl);
+                    }
                 }
             }
         }
@@ -1654,6 +1687,7 @@ export class NeedleXRSession implements INeedleXRSession {
     }
 
     private invokeCallback_EnterXR(script: INeedleXRSessionEventReceiver) {
+        this._scripts_in_xr.add(script);
         if (script.onEnterXR) {
             script.onEnterXR({ xr: this });
         }
@@ -1669,6 +1703,7 @@ export class NeedleXRSession implements INeedleXRSession {
         }
     }
     private invokeCallback_LeaveXR(script: INeedleXRSessionEventReceiver) {
+        if (!this._scripts_in_xr.delete(script)) return;
         if (script.onLeaveXR && !script.destroyed) {
             script.onLeaveXR({ xr: this });
         }

package/src/engine-components/Animation.ts

@@ -447,25 +447,13 @@ export class Animation extends Behaviour implements IAnimationComponent {
 
     private internalOnPlay(action: AnimationAction, options: PlayOptions): Promise<AnimationAction> {
         var existing = this.actions.find(a => a === action);
-        if (existing === action && existing.isRunning() && existing.time < existing.getClip().duration) {
-            const handle = this.tryFindHandle(action);
-            if (existing.paused) {
-                existing.paused = false;
-            }
-            if (handle) return handle.waitForFinish();
-        }
 
-        // Assign defaults
-        if (options.loop === undefined) options.loop = this.loop;
-        if (options.clampWhenFinished === undefined) options.clampWhenFinished = this.clampWhenFinished;
-        if (options.minMaxOffsetNormalized === undefined && this.randomStartTime) options.minMaxOffsetNormalized = this.minMaxOffsetNormalized;
-        if (options.minMaxSpeed === undefined) options.minMaxSpeed = this.minMaxSpeed;
-
-        // Reset currently running animations
+        // Stop other animations before the early-return check so exclusive always applies,
+        // even when the target animation is already running.
         const stopOther = options?.exclusive ?? true;
         if (stopOther) {
             for (const act of this.actions) {
-                if (act != existing) {
+                if (act !== action) {
                     if (options.fadeDuration) {
                         act.fadeOut(options.fadeDuration);
                     }
@@ -475,6 +463,20 @@ export class Animation extends Behaviour implements IAnimationComponent {
                 }
             }
         }
+
+        if (existing === action && existing.isRunning() && existing.time < existing.getClip().duration) {
+            const handle = this.tryFindHandle(action);
+            if (existing.paused) {
+                existing.paused = false;
+            }
+            if (handle) return handle.waitForFinish();
+        }
+
+        // Assign defaults
+        if (options.loop === undefined) options.loop = this.loop;
+        if (options.clampWhenFinished === undefined) options.clampWhenFinished = this.clampWhenFinished;
+        if (options.minMaxOffsetNormalized === undefined && this.randomStartTime) options.minMaxOffsetNormalized = this.minMaxOffsetNormalized;
+        if (options.minMaxSpeed === undefined) options.minMaxSpeed = this.minMaxSpeed;
         if (options?.fadeDuration) {
             action.fadeIn(options.fadeDuration);
         }
@@ -515,10 +517,11 @@ export class Animation extends Behaviour implements IAnimationComponent {
 
         action.paused = false;
         action.play();
+        if (debug) console.log("[Animation] Now Playing", action.getClip().name, action)
+
 
         window.requestAnimationFrame(() => AnimationUtils.testIfRootCanAnimate(action));
 
-        if (debug) console.log("PLAY", action.getClip().name, action)
         const handle = new AnimationHandle(action, this.mixer!, options, _ => {
             this._handles.splice(this._handles.indexOf(handle), 1);
         });

package/src/engine-components/AnimatorController.ts

@@ -1316,4 +1316,7 @@ class AnimatorControllerSerializator extends TypeSerializer {
         return undefined;
     }
 }
-new AnimatorControllerSerializator(AnimatorController);
+/** @internal */
+export function initAnimatorControllerSerializer() {
+    new AnimatorControllerSerializator(AnimatorController);
+}