@needle-tools/engine 2.24.0-pre

package/src/engine-components/AudioListener.ts

@@ -0,0 +1,32 @@
+import { Behaviour, GameObject } from "./Component";
+import * as THREE from "three";
+import { AudioSource } from "./AudioSource";
+import { Camera } from "./Camera";
+
+
+export class AudioListener extends Behaviour {
+
+    get listener(): THREE.AudioListener {
+        if (this._listener == null)
+            this._listener = new THREE.AudioListener();
+        return this._listener;
+    }
+
+    private _listener: THREE.AudioListener | null = null;
+
+    awake() {
+        AudioSource.registerWaitForAllowAudio(() => {
+            const listener = this.listener;
+            if (listener == null) return;
+            // if the listener is already parented to some object d0nt change it
+            if (listener.parent) return;
+
+            const cam = GameObject.getComponentInParent(this.gameObject, Camera);
+            if (cam) {
+                cam.cam.add(listener);
+            }
+            else
+                this.gameObject.add(listener);
+        });
+    }
+}

package/src/engine-components/AudioSource.ts

@@ -0,0 +1,383 @@
+import { Behaviour, GameObject } from "./Component";
+import * as THREE from "three";
+import { PositionalAudioHelper } from 'three/examples/jsm/helpers/PositionalAudioHelper.js';
+import { AudioListener } from "./AudioListener";
+import * as utils from "../engine/engine_utils";
+import { serializeable } from "../engine/engine_serialization_decorator";
+import { Application, ApplicationEvents } from "../engine/engine_application";
+
+
+const debug = utils.getParam("debugaudio");
+
+export enum AudioRolloffMode {
+    /// <summary>
+    ///   <para>Use this mode when you want a real-world rolloff.</para>
+    /// </summary>
+    Logarithmic = 0,
+    /// <summary>
+    ///   <para>Use this mode when you want to lower the volume of your sound over the distance.</para>
+    /// </summary>
+    Linear = 1,
+    /// <summary>
+    ///   <para>Use this when you want to use a custom rolloff.</para>
+    /// </summary>
+    Custom = 2,
+}
+
+
+export class AudioSource extends Behaviour {
+
+    private static _didCallBeginWaitForUserInteraction: boolean = false;
+    public static get userInteractionRegistered(): boolean {
+
+        if (!AudioSource._didCallBeginWaitForUserInteraction) {
+            AudioSource._didCallBeginWaitForUserInteraction = true;
+            AudioSource._beginWaitForUserInteraction();
+        }
+        return AudioSource._userInteractionRegistered;
+    }
+
+    private static callbacks: Function[] = [];
+    public static registerWaitForAllowAudio(cb: Function) {
+        if (cb !== null) {
+            if (this._userInteractionRegistered) {
+                cb();
+                return;
+            }
+            if (this.callbacks.indexOf(cb) === -1)
+                this.callbacks.push(cb);
+            if (!AudioSource._didCallBeginWaitForUserInteraction) {
+                AudioSource._didCallBeginWaitForUserInteraction = true;
+                AudioSource._beginWaitForUserInteraction();
+            }
+        }
+    }
+
+    private static _userInteractionRegistered: boolean = false;
+    private static _beginWaitForUserInteraction(cb: Function | null = null) {
+        if (this._userInteractionRegistered) {
+            if (cb) cb();
+            return;
+        }
+        if (cb !== null)
+            this.registerWaitForAllowAudio(cb);
+        const callback = () => {
+            if (fn == undefined) return;
+            if (AudioSource._userInteractionRegistered) return;
+            AudioSource._userInteractionRegistered = true;
+            console.log("registered interaction, can play audio now");
+            document.removeEventListener('pointerdown', fn);
+            document.removeEventListener('click', fn);
+            document.removeEventListener('dragstart', fn);
+            document.removeEventListener('touchstart', fn);
+            for (const cb of this.callbacks) {
+                cb();
+            }
+            this.callbacks.length = 0;
+        };
+        let fn = callback.bind(this);
+        document.addEventListener('pointerdown', fn);
+        document.addEventListener('click', fn);
+        document.addEventListener('dragstart', fn);
+        document.addEventListener('touchstart', fn);
+    }
+
+    @serializeable()
+    private clip: string = "";
+
+    @serializeable()
+    playOnAwake: boolean = false;
+
+    @serializeable()
+    get loop(): boolean {
+        if (this.sound) this._loop = this.sound.getLoop();
+        return this._loop;
+    }
+    set loop(val: boolean) {
+        this._loop = val;
+        if (this.sound) this.sound.setLoop(val);
+    }
+    @serializeable()
+    get spatialBlend(): number {
+        return this._spatialBlend;
+    }
+    set spatialBlend(val: number) {
+        if (val === this._spatialBlend) return;
+        this._spatialBlend = val;
+        this._needUpdateSpatialDistanceSettings = true;
+    }
+    @serializeable()
+    get minDistance(): number {
+        return this._minDistance;
+    }
+    set minDistance(val: number) {
+        if (this._minDistance === val) return;
+        this._minDistance = val;
+        this._needUpdateSpatialDistanceSettings = true;
+    }
+    @serializeable()
+    get maxDistance(): number {
+        return this._maxDistance;
+    }
+    set maxDistance(val: number) {
+        if (this._maxDistance === val) return;
+        this._maxDistance = val;
+        this._needUpdateSpatialDistanceSettings = true;
+    }
+
+    private _spatialBlend: number = 0;
+    private _minDistance: number = 1;
+    private _maxDistance: number = 100;
+
+    @serializeable()
+    get volume(): number { return this._volume; }
+    set volume(val: number) {
+        this._volume = val;
+        if (this.sound) {
+            if (debug) console.log(this.name, "audio set volume", val);
+            this.sound.setVolume(val);
+        }
+    }
+    private _volume: number = 1;
+
+    @serializeable()
+    rollOffMode: AudioRolloffMode = 0;
+
+
+    private _loop: boolean = false;
+    private sound: THREE.PositionalAudio | null = null;
+    private helper: PositionalAudioHelper | null = null;
+    private wasPlaying = false;
+    private audioLoader: THREE.AudioLoader | null = null;
+    private shouldPlay: boolean = false;
+    // set this from audio context time, used to set clip offset when setting "time" property
+    // there is maybe a better way to set a audio clip current time?!
+    private _lastClipStartedLoading: string | null = null;
+
+    public get Sound(): THREE.PositionalAudio | null {
+        if (!this.sound && AudioSource._userInteractionRegistered) {
+            const listener = GameObject.getComponent(this.context.mainCamera, AudioListener) ?? GameObject.findObjectOfType(AudioListener, this.context);
+            if (listener?.listener) {
+                this.sound = new THREE.PositionalAudio(listener.listener);
+                this.gameObject.add(this.sound);
+            }
+        }
+        return this.sound;
+    }
+
+    public get ShouldPlay(): boolean { return this.shouldPlay; }
+
+    private _focusCallback?: any;
+
+    awake() {
+        this.audioLoader = new THREE.AudioLoader();
+        if (this.playOnAwake) this.shouldPlay = true;
+
+        window.addEventListener('visibilitychange', _evt => {
+            switch (document.visibilityState) {
+                case "hidden":
+                    this.wasPlaying = this.isPlaying;
+                    this.pause();
+                    break;
+                case "visible":
+                    if (this.wasPlaying) this.play();
+                    break;
+            }
+        });
+
+        this._focusCallback = () => {
+            if (this.enabled && this.playOnAwake && !this.isPlaying && AudioSource._userInteractionRegistered) {
+                this.play();
+            }
+        };
+
+        this.context.application.addEventListener(ApplicationEvents.Visible, this._focusCallback);
+    }
+
+    onDestroy() {
+        this.context.application.removeEventListener(ApplicationEvents.Visible, this._focusCallback);
+    }
+
+
+    onEnable(): void {
+        if (!AudioSource._userInteractionRegistered) {
+            AudioSource._beginWaitForUserInteraction(() => {
+                if (this.enabled && !this.destroyed && this.shouldPlay)
+                    this.loadAndPlay(this.clip);
+            });
+        }
+        else if (this.playOnAwake && this.context.application.isVisible) {
+            this.play();
+        }
+    }
+
+    onDisable() {
+        this.stop();
+    }
+
+    private lerp = (x, y, a) => x * (1 - a) + y * a;
+
+    private onLoaded(buffer) {
+        if (debug) console.log("audio buffer loaded");
+        AudioSource.registerWaitForAllowAudio(() => {
+            if (debug)
+                console.log("finished loading", buffer);
+
+            const sound = this.Sound;
+            if (!sound) {
+                console.warn("Failed getting sound", this.name);
+                return;
+            }
+            if (sound.isPlaying)
+                sound.stop();
+
+            sound.setBuffer(buffer);
+            sound.loop = this.loop;
+            sound.setVolume(this.volume);
+            sound.autoplay = this.shouldPlay;
+            // sound.setDistanceModel('linear');
+            // sound.setRolloffFactor(1);
+            this.applySpatialDistanceSettings();
+            // sound.setDirectionalCone(180, 360, 0.1);
+            if (sound.isPlaying)
+                sound.stop();
+
+            if (debug) console.log(this.name, this.shouldPlay, AudioSource.userInteractionRegistered, this);
+
+            if (this.shouldPlay && AudioSource._userInteractionRegistered)
+                this.play();
+        });
+    }
+
+    private applySpatialDistanceSettings() {
+        const sound = this.sound;
+        if (!sound) return;
+        this._needUpdateSpatialDistanceSettings = false;
+        const dist = this.lerp(10 * this._maxDistance / Math.max(0.0001, this.spatialBlend), this._minDistance, this.spatialBlend);
+        if (debug) console.log(this.name, this._minDistance, this._maxDistance, this.spatialBlend, "Ref distance=" + dist);
+        sound.setRefDistance(dist);
+        sound.setMaxDistance(Math.max(0.01, this._maxDistance));
+        // https://developer.mozilla.org/en-US/docs/Web/API/PannerNode/distanceModel
+        switch (this.rollOffMode) {
+            case AudioRolloffMode.Logarithmic:
+                sound.setDistanceModel('exponential');
+                break;
+            case AudioRolloffMode.Linear:
+                sound.setDistanceModel('linear');
+                break;
+            case AudioRolloffMode.Custom:
+                break;
+        }
+
+        if (this.spatialBlend > 0) {
+            if (debug && !this.helper) {
+                this.helper = new PositionalAudioHelper(sound, sound.getRefDistance());
+                sound.add(this.helper);
+            }
+        }
+        else if (this.helper && this.helper.parent) {
+            this.helper.removeFromParent();
+        }
+    }
+
+    private loadAndPlay(clip?: string) {
+        if (clip)
+            this.clip = clip;
+        if (this.clip) {
+            if (debug)
+                console.log(this.clip);
+            if (this.clip.endsWith(".mp3") || this.clip.endsWith(".wav")) {
+                if (!this.audioLoader)
+                    this.audioLoader = new THREE.AudioLoader();
+                this.shouldPlay = true;
+                if (this._lastClipStartedLoading === this.clip) {
+                    if (debug) console.log("Is currently loading:", this._lastClipStartedLoading, this)
+                    return;
+                }
+                this._lastClipStartedLoading = this.clip;
+                if (debug)
+                    console.log("load audio", this.clip);
+                this.audioLoader.load(this.clip, this.onLoaded.bind(this), () => { }, console.error);
+            }
+        }
+    }
+
+    play(clip: string | undefined = undefined) {
+        if (!this.audioLoader || !this.sound || (clip && clip !== this.clip)) {
+            this.loadAndPlay(clip);
+            return;
+        }
+
+        this.shouldPlay = true;
+        this._hasEnded = false;
+        if (debug)
+            console.log("play", this.sound?.getVolume(), this.sound);
+        if (this.sound && !this.sound.isPlaying) {
+            this.sound.play();
+        }
+    }
+
+    pause() {
+        if (debug) console.log("Pause", this);
+        this._hasEnded = true;
+        this.shouldPlay = false;
+        if (this.sound && this.sound.isPlaying) {
+            this._lastContextTime = this.sound?.context.currentTime;
+            this.sound.pause();
+        }
+    }
+
+    stop() {
+        if (debug) console.log("Pause", this);
+        this._hasEnded = true;
+        this.shouldPlay = false;
+        if (this.sound) {
+            this._lastContextTime = this.sound?.context.currentTime;
+            if (debug)
+                console.log(this._lastContextTime)
+            this.sound.stop();
+        }
+    }
+
+    private _lastContextTime: number = 0;
+
+    get isPlaying(): boolean { return this.sound?.isPlaying ?? false; }
+    set isPlaying(_: boolean) { }
+
+    get time(): number { return this.sound?.source ? (this.sound.source?.context.currentTime - this._lastContextTime + this.sound.offset) : 0; }
+    set time(val: number) {
+        if (this.sound) {
+            if (val === this.sound.offset) return;
+            const wasPlaying = this.isPlaying;
+            this.stop();
+            this.sound.offset = val;
+            if (wasPlaying)
+                this.play();
+        }
+    }
+
+    private _hasEnded: boolean = true;
+    private _needUpdateSpatialDistanceSettings: boolean = false;
+
+    update() {
+        if (this.helper) {
+            if (this.isPlaying)
+                this.helper.update();
+            this.helper.visible = this.isPlaying;
+        }
+
+        if (this._needUpdateSpatialDistanceSettings) {
+            this.applySpatialDistanceSettings();
+        }
+
+        if (this.sound && !this.sound.isPlaying && this.shouldPlay && !this._hasEnded) {
+            this._hasEnded = true;
+            if (debug)
+                console.log("Audio clip ended", this.clip);
+            this.sound.dispatchEvent({ type: 'ended', target: this });
+        }
+
+        // this.gameObject.position.x = Math.sin(time.time) * 2;
+        // this.gameObject.position.z = Math.cos(time.time * .5) * 2;
+    }
+}

package/src/engine-components/AvatarLoader.ts

@@ -0,0 +1,240 @@
+import { GLTF, GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader";
+import * as THREE from "three";
+import * as utils from "../engine/engine_utils"
+import * as object from "../engine/engine_gltf_builtin_components";
+import * as loaders from "../engine/engine_loaders"
+import { Context } from "../engine/engine_setup";
+import { GameObject, InstantiateOptions } from "./Component";
+import { download_file } from "../engine/engine_web_api";
+import { parseSync } from "../engine/engine_scenetools";
+
+const debug = utils.getParam("debugavatar");
+
+export class AvatarModel {
+    root: THREE.Object3D;
+    head: THREE.Object3D;
+    leftHand: THREE.Object3D | null;
+    rigthHand: THREE.Object3D | null;
+
+
+    get isValid(): boolean {
+        return this.head !== null && this.head !== undefined;
+    }
+
+    constructor(root: THREE.Object3D, head: THREE.Object3D, leftHand: THREE.Object3D | null, rigthHand: THREE.Object3D | null) {
+        this.root = root;
+        this.head = head;
+        this.leftHand = leftHand;
+        this.rigthHand = rigthHand;
+        this.root?.traverse(h => h.layers.set(2));
+        // this.head?.traverse(h => h.layers.set(2));
+        // this.leftHand?.traverse(h => h.layers.set(2));
+        // this.rigthHand?.traverse(h => h.layers.set(2));
+    }
+
+    // createNewInstance(): AvatarModel {
+    //     const head = GameObject.instantiate(this.head);
+    //     const lh = GameObject.instantiate(this.leftHand);
+    //     const rh = GameObject.instantiate(this.rigthHand);
+    //     const model = new AvatarModel(this.root, head, lh, rh);
+    //     this.assignRandomColors();
+    //     return model;
+    // }
+
+    assignRandomColors() {
+        AvatarLoader.assignRandomPlayerColors(this);
+    }
+}
+
+export class AvatarLoader {
+
+    private readonly avatarRegistryUrl: string | null = null;// = "https://needle-storage-castle-demo.glitch.me";//"https://smol-worlds.glitch.me/files/";
+    // private loader: GLTFLoader | null;
+    private avatarModelCache: Map<string, AvatarModel | null> = new Map<string, AvatarModel | null>();
+
+    public async getOrCreateNewAvatarInstance(context: Context, avatarId: string | THREE.Object3D): Promise<AvatarModel | null> {
+
+        if (!avatarId) {
+            console.error("Can not create avatar: failed to provide id or root object");
+            return null;
+        }
+
+        let root: THREE.Object3D | null = null;
+        if (typeof avatarId === "string") {
+            root = await this.loadAvatar(context, avatarId);
+            if (!root) {
+                const opts = new InstantiateOptions();
+                // opts.parent = context.scene.uuid;
+                root = GameObject.instantiate(utils.tryFindObject(avatarId, context.scene), opts);
+            }
+        }
+        else root = avatarId;
+
+        if (!root) {
+            return null;
+        }
+        const model = this.findAvatar(root);
+        // console.log(model);
+        model.assignRandomColors();
+        // this.cacheModel(avatarId, model);
+
+        if (model.isValid) {
+            if (debug)
+                console.log("[Custom Avatar] valid config", avatarId, debug ? model : "");
+            return model;
+        }
+        else {
+            console.warn("[Custom Avatar] config isn't valid", avatarId, debug ? model : "");
+            return null;
+        }
+    }
+
+
+    private async loadAvatar(context: Context, avatarId: string): Promise<THREE.Object3D | null> {
+
+        console.assert(avatarId !== undefined && avatarId !== null && typeof avatarId === "string", "Avatar id must not be null");
+        if (avatarId.length <= 0) return null;
+        if (!avatarId) return null;
+
+        if (debug)
+            console.log("[Custom Avatar] " + avatarId + ", loading...");
+        // should probably be done by the server?!
+        if (!avatarId.endsWith(".glb"))
+            avatarId += ".glb";
+
+
+        // for the demo we use the storage backend we uploaded the avatar to (by file drop)
+        if (this.avatarRegistryUrl === null) {
+            // try loading avatar from local file
+            const download_res = await fetch("./" + avatarId);
+            let bin: ArrayBuffer | null = null;
+            if (download_res.ok) {
+                const blob = await download_res.blob();
+                if (blob) bin = await blob.arrayBuffer();
+            }
+            if (!bin) {
+                bin = await download_file(avatarId, avatarId, 0, "no url here go away", true);
+                if (!bin) return null;
+            }
+
+            const gltf = await parseSync(context, bin, null!, 0);
+            return gltf?.scene ?? null;
+        }
+
+
+        // if (this.avatarModelCache.has(avatarId)) {
+        //     console.log("[Custom Avatar] " + avatarId + ", found in cache");
+        //     return new Promise((res, _) => {
+        //         var model = this.avatarModelCache.get(avatarId)?.createNewInstance();
+        //         res(model);
+        //     });
+        // }
+        // return null;
+
+        const loader = new GLTFLoader();
+        loaders.addDracoAndKTX2Loaders(loader, context);
+
+        // TODO: cache binary (fetch finary from model gallery and use binary method)
+        return new Promise((resolve, _reject) => {
+            const url = this.avatarRegistryUrl + "/" + avatarId;
+            loader.load(url,
+                async gltf => {
+                    await object.createBuiltinComponents(context, url, gltf);
+                    resolve(gltf.scene);
+                },
+                progress => {
+                    if (debug)
+                        console.log("[Custom Avatar] " + (progress.loaded / progress.total * 100) + '% loaded of ' + (progress.total / 1024) + "kB");
+                },
+                error => {
+                    console.error("[Custom Avatar] " + "Error when loading: " + error);
+                    resolve(null);
+                }
+            );
+        });
+    }
+
+    private cacheModel(_id: string, _model: AvatarModel) {
+        // this.avatarModelCache.set(id, model);
+    }
+
+    // TODO this should be burned to the ground once 🤞 we have proper extras that define object relations.
+    private findAvatar(obj: THREE.Object3D): AvatarModel {
+
+        const root: THREE.Object3D = obj;
+        let searchIn = root;
+        // some GLTFs have a "scene" root it seems, others don't, we skip the root here if there's only one child
+        if (searchIn.children.length == 1)
+            searchIn = obj.children[0];
+        let head = this.findAvatarPart(searchIn, "head");
+
+        const leftHand = this.findAvatarPart(searchIn, "left");
+        const rightHand = this.findAvatarPart(searchIn, "right");
+
+        if (!head) {
+            // very last fallback, entire root is used as head
+            head = root;
+
+            // normalize size, if the object isn't properly setup the scale might be totally off
+            let boundsSize = new THREE.Vector3();
+            new THREE.Box3().setFromObject(head).getSize(boundsSize);
+            let maxAxis = Math.max(boundsSize.x, boundsSize.y, boundsSize.z);
+            console.warn("[Custom Avatar] " + "Normalizing head scale, it's too big: " + maxAxis + " meters! Should be < 0.3m");
+            if (maxAxis > 0.3) {
+                head.scale.multiplyScalar(1.0 / maxAxis * 0.3);
+            }
+        }
+
+        const model = new AvatarModel(root, head, leftHand, rightHand);
+        return model;
+    }
+
+
+    private findAvatarPart(obj: THREE.Object3D, searchString: string): THREE.Object3D | null {
+        for (let child of obj.children) {
+            if (child.name.toLowerCase().indexOf(searchString) > -1)
+                return child;
+        }
+        return null;
+    }
+
+    private handleCustomAvatarErrors(response) {
+        if (!response.ok) {
+            throw Error(response.statusText);
+        }
+        return response;
+    }
+
+    public static assignRandomPlayerColors(model: AvatarModel) {
+
+        const foundMaterials: Map<string, THREE.Material> = new Map<string, THREE.Material>();
+
+        function findPlayerColorMeshesAndAssignColors(o) {
+            // console.log(o);
+            if (o.type === "Mesh") {
+                const mesh = o as THREE.Mesh;
+                const mat: THREE.Material = mesh.material as THREE.Material;
+                if (mat && mat.name) {
+                    if (foundMaterials.has(mat.uuid)) {
+                        const found = foundMaterials.get(mat.uuid);
+                        if (found)
+                            mesh.material = found;
+                        return;
+                    }
+                    if (mat.name.endsWith("_playercolor") || mat.name.endsWith("_player_color2")) {
+                        const uuid = mat.uuid;
+                        mesh.material = mat.clone();
+                        mesh.material["color"] = new THREE.Color(Math.random(), Math.random(), Math.random());
+                        foundMaterials.set(uuid, mesh.material);
+                    }
+                }
+            }
+        }
+        if (model.head)
+            model.head.traverse(findPlayerColorMeshesAndAssignColors);
+        if (model.leftHand)
+            model.leftHand.traverse(findPlayerColorMeshesAndAssignColors);
+        if (model.rigthHand)
+            model.rigthHand.traverse(findPlayerColorMeshesAndAssignColors);
+    }
+}

package/src/engine-components/AxesHelper.ts

@@ -0,0 +1,32 @@
+import { Behaviour } from "./Component";
+import * as params from "../engine/engine_default_parameters";
+import { serializeable } from "../engine/engine_serialization_decorator";
+import { AxesHelper as _AxesHelper } from "three";
+
+export class AxesHelper extends Behaviour {
+    @serializeable()
+    public length: number = 1;
+    @serializeable()
+    public depthTest: boolean = true;
+    @serializeable()
+    public isGizmo:boolean = true;
+
+    private _axes: THREE.AxesHelper | null = null;
+
+    onEnable(): void {
+        if (this.isGizmo && !params.showGizmos) return;
+        if (!this._axes)
+            this._axes = new _AxesHelper(this.length);
+        this.gameObject.add(this._axes);
+        const mat: any = this._axes.material;
+        if (mat) {
+            if (mat.depthTest !== undefined)
+                mat.depthTest = this.depthTest;
+        }
+    }
+
+    onDisable(): void {
+        if (!this._axes) return;
+        this.gameObject.remove(this._axes);
+    }
+}