@needle-tools/engine 2.66.1-pre → 2.67.0-pre

package/src/engine/engine_types.ts

@@ -1,3 +1,4 @@
+import { RenderTexture } from "./engine_texture";
 import { Camera, Color, Material, Object3D, Vector3, Quaternion, Ray, Scene, Renderer, WebGLRenderer } from "three";
 import { RGBAColor } from "../engine-components/js-extensions/RGBAColor";
 import { CollisionDetectionMode, PhysicsMaterial, RigidbodyConstraints } from "./engine_physics.types";
@@ -72,12 +73,14 @@ export declare interface IGameObject extends Object3D {
 
     activeSelf: boolean;
 
+    destroy(): void;
+
     /** NOTE: this is just a wrapper for devs coming from Unity. Please use this.gameObject instead. In Needle Engine this.gameObject is the same as this.gameObject.transform. See the tutorial link below for more information
      * @augments Object3D
      * @tutorial https://fwd.needle.tools/needle-engine/docs/transform
      * */
     get transform(): Object3D;
-    
+
     addNewComponent<T>(type: Constructor<T>): T | null;
     removeComponent(comp: IComponent): IComponent;
     getOrAddComponent<T>(typeName: Constructor<T> | null): T;
@@ -161,6 +164,7 @@ export interface IComponent {
 export declare interface ICamera extends IComponent {
     get isCamera(): boolean;
     applyClearFlagsIfIsActiveCamera(): unknown;
+    applyClearFlags();
     buildCamera();
     get cam(): Camera;
     nearClipPlane: number;
@@ -171,6 +175,7 @@ export declare interface ICamera extends IComponent {
     aspect: number;
     fieldOfView?: number;
     screenPointToRay(x: number, y: number, ray?: Ray): Ray;
+    targetTexture: RenderTexture | null;
 }
 
 export declare interface ICameraController {

package/src/engine/engine_utils.ts

@@ -44,8 +44,9 @@ export class CircularBuffer<T> {
 let saveParams: boolean = false;
 const requestedParams: Array<string> = [];
 setTimeout(() => {
-    if (saveParams)
-        console.log(requestedParams);
+    if (saveParams) {
+        console.log(requestedParams.sort());
+    }
 }, 100);
 
 export function getUrlParams() {
@@ -61,7 +62,7 @@ export function getParam(paramName: string): string | boolean | number {
         const val = urlParams.get(paramName);
         if (val) {
             const num = Number(val);
-            if (!isNaN(num)) return num; 
+            if (!isNaN(num)) return num;
             return val;
         }
         else return true;
@@ -83,10 +84,10 @@ export function setParam(paramName: string, paramValue: string): void {
 export function setParamWithoutReload(paramName: string, paramValue: string | null, appendHistory = true): void {
     const urlParams = getUrlParams();
     if (urlParams.has(paramName)) {
-        if(paramValue === null) urlParams.delete(paramName);
+        if (paramValue === null) urlParams.delete(paramName);
         else urlParams.set(paramName, paramValue);
     }
-    else if(paramValue !== null)
+    else if (paramValue !== null)
         urlParams.append(paramName, paramValue);
     if (appendHistory) pushState(paramName, urlParams);
     else setState(paramName, urlParams);

package/src/engine/extensions/NEEDLE_progressive.ts

@@ -13,6 +13,7 @@ declare type ProgressiveTextureSchema = {
     uri: string;
     guid: string;
 }
+const $progressiveTextureExtension = Symbol("needle-progressive-texture");
 
 
 const debug_toggle_maps: Map<Material, { [key: string]: { original: Texture, lod0: Texture } }> = new Map();
@@ -35,6 +36,7 @@ if (debug) {
     });
 }
 
+
 export class NEEDLE_progressive implements GLTFLoaderPlugin {
 
     static assignTextureLOD(context: Context, source: SourceIdentifier | undefined, material: Material, level: number = 0) {
@@ -86,21 +88,6 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
         this.context = context;
     }
 
-    private _loading: number[] = [];
-
-    // beforeRoot(): null {
-    //     console.log("BEFORE ROOT", this.parser);
-    //     return null;
-    // }
-
-    // loadTexture(index: number): Promise<Texture> | null {
-    //     console.log(index, this._loading);
-    //     if (this._loading.includes(index)) return null;
-    //     const textureInfo = this.parser.json.textures[index];
-    //     if (debug)
-    //         console.log(index, textureInfo);
-    //     return null;
-    // }
 
     afterRoot(gltf: GLTF): null {
         if (debug)
@@ -110,10 +97,12 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                 const ext: ProgressiveTextureSchema = textureInfo?.extensions[EXTENSION_NAME];
                 if (ext) {
                     const prom = this.parser.getDependency("texture", index);
-                    this._loading.splice(this._loading.indexOf(index), 1);
                     prom.then(t => {
-                        if (debug) console.log("register texture", t.name, t.uuid, ext);
-                        t.userData.deferred = ext;
+                        if (debug) console.log("> Progressive: register", t.name, t.uuid, ext);
+                        // Put the extension info into the source (seems like tiled textures are cloned and the userdata etc is not properly copied BUT the source of course is not cloned)
+                        // see https://github.com/needle-tools/needle-engine-support/issues/133
+                        if (t.source)
+                            t.source[$progressiveTextureExtension] = ext;
                         NEEDLE_progressive.cache.set(t.uuid, ext);
                     });
                 }
@@ -130,17 +119,24 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
     private static async getOrLoadTexture(context: Context, source: SourceIdentifier | undefined, material: Material, slot: string, current: Texture, _level: number): Promise<Texture | null> {
 
         const key = current.uuid;
-        const ext: ProgressiveTextureSchema | undefined = NEEDLE_progressive.cache.get(key);// || current.userData.deferred;
-        if (ext) {
+
+        let progressiveInfo: ProgressiveTextureSchema | undefined;
+
+        // See https://github.com/needle-tools/needle-engine-support/issues/133
+        if(current.source && current.source[$progressiveTextureExtension])
+            progressiveInfo = current.source[$progressiveTextureExtension];
+        if(!progressiveInfo) progressiveInfo = NEEDLE_progressive.cache.get(key);
+
+        if (progressiveInfo) {
             if (debug)
-                console.log(key, ext.uri, ext.guid);
-            const uri = getPath(source, ext.uri);
+                console.log(key, progressiveInfo.uri, progressiveInfo.guid);
+            const uri = getPath(source, progressiveInfo.uri);
             if (uri.endsWith(".glb") || uri.endsWith(".gltf")) {
-                if (!ext.guid) {
-                    console.warn("missing pointer for glb/gltf texture", ext);
+                if (!progressiveInfo.guid) {
+                    console.warn("missing pointer for glb/gltf texture", progressiveInfo);
                     return null;
                 }
-                const resolveKey = uri + "_" + ext.guid;
+                const resolveKey = uri + "_" + progressiveInfo.guid;
                 if (this.resolved[resolveKey]) {
                     if (debug) console.log("Texture has already been loaded: " + resolveKey, material.name, slot, current.name);
                     return this.resolved[resolveKey];
@@ -148,16 +144,18 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
 
                 const info = this.onProgressiveLoadStart(context, source, uri, material, slot);
                 try {
-                    if(this.currentlyLoading[resolveKey] !== undefined) {
-                        if(debug)
+                    if (this.currentlyLoading[resolveKey] !== undefined) {
+                        if (debug)
                             console.log("Already loading:", material.name + "." + slot, resolveKey);
                         const tex = await this.currentlyLoading[resolveKey];
                         return tex;
                     }
+                    const ext = progressiveInfo;
                     const request = new Promise<Texture | null>(async (resolve, _) => {
                         const loader = new GLTFLoader();
                         addDracoAndKTX2Loaders(loader, context);
 
+
                         if (debug) console.log("Load " + uri, material.name, slot, ext.guid);
                         if (debug) {
                             await delay(Math.random() * 1000);
@@ -183,10 +181,12 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                         if (!found)
                             return resolve(null);
 
-                        const tex = await parser.getDependency("texture", index);
-                        tex.encoding = current.encoding;
+                        const tex = await parser.getDependency("texture", index) as Texture;
+                        const source = tex.source;
+                        tex.copy(current);
+                        tex.source = source;
                         if (tex) {
-                            tex.guid = ext.guid;
+                            (tex as any).guid = ext.guid;
                         }
                         this.resolved[resolveKey] = tex as Texture;
                         if (debug)
@@ -209,12 +209,12 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                     const loader = new TextureLoader();
                     const tex = await loader.loadAsync(uri);
                     if (tex) {
-                        (tex as any).guid = ext.guid;
+                        (tex as any).guid = progressiveInfo.guid;
                         tex.flipY = false;
                         tex.needsUpdate = true;
                         tex.encoding = current.encoding;
                         if (debug)
-                            console.log(ext, tex);
+                            console.log(progressiveInfo, tex);
                     }
                     else if (debug) console.warn("failed loading", uri);
                     return tex;

package/src/engine/extensions/extensions.ts

@@ -8,11 +8,26 @@ import { NEEDLE_gameobject_data } from "./NEEDLE_gameobject_data";
 import { NEEDLE_persistent_assets } from "./NEEDLE_persistent_assets";
 // import { KHR_animation_pointer } from "./KHR_animation_pointer";
 import { NEEDLE_lightmaps } from "../extensions/NEEDLE_lightmaps";
-import { SourceIdentifier } from "../engine_types";
+import { Constructor, ConstructorConcrete, SourceIdentifier } from "../engine_types";
 import { Context } from "../engine_setup";
 import { NEEDLE_lighting_settings } from "./NEEDLE_lighting_settings";
 import { NEEDLE_render_objects } from "./NEEDLE_render_objects";
 import { NEEDLE_progressive } from "./NEEDLE_progressive";
+import { InternalUsageTrackerPlugin } from "./usage_tracker";
+import { isUsageTrackingEnabled } from "../engine_assetdatabase";
+import { GLTFLoaderPlugin } from "three/examples/jsm/loaders/GLTFLoader";
+
+const _addedCustomExtension = new Array<ConstructorConcrete<GLTFLoaderPlugin>>();
+
+export function addCustomExtension(ext: ConstructorConcrete<GLTFLoaderPlugin>) {
+    if (!_addedCustomExtension.includes(ext))
+        _addedCustomExtension.push(ext);
+}
+export function removeCustomExtension(ext: ConstructorConcrete<GLTFLoaderPlugin>) {
+    const index = _addedCustomExtension.indexOf(ext);
+    if (index >= 0)
+        _addedCustomExtension.splice(index, 1);
+}
 
 export function registerComponentExtension(loader: GLTFLoader): NEEDLE_components {
     const ext = new NEEDLE_components();
@@ -23,6 +38,8 @@ export function registerComponentExtension(loader: GLTFLoader): NEEDLE_component
     return ext;
 }
 
+
+
 class PointerResolver {
     resolvePath(path: string) {
         if (path.includes('/extensions/builtin_components/'))
@@ -42,6 +59,10 @@ export function registerExtensions(loader: GLTFLoader, context: Context, sourceI
     loader.register(p => new NEEDLE_render_objects(p, sourceId));
     loader.register(p => new NEEDLE_progressive(p, sourceId, context));
     loader.register(p => new EXT_texture_exr(p));
+    if (isUsageTrackingEnabled()) loader.register(p => new InternalUsageTrackerPlugin(p))
+
+    for (const ext of _addedCustomExtension)
+        loader.register(p => new ext(p));
 
     const setPointerResolverFunction = loader["setAnimationPointerResolver"];
     if (typeof setPointerResolverFunction === "function")

package/src/engine/extensions/usage_tracker.ts

@@ -0,0 +1,91 @@
+
+import { GLTF, GLTFLoaderPlugin, GLTFParser } from "three/examples/jsm/loaders/GLTFLoader";
+import { Group, Mesh, Object3D, SkinnedMesh } from "three";
+import { getParam } from "../engine_utils";
+
+
+const $loadingId = Symbol("gltf-loader-internal-usage-tracker");
+const debug = getParam("debugusers");
+
+export class InternalUsageTrackerPlugin implements GLTFLoaderPlugin {
+
+    static isLoading(object: object) {
+        return InternalUsageTrackerPlugin._loadingProcesses > 0;
+        return object[$loadingId] !== undefined;
+    }
+    private static _loadingProcesses = 0;
+
+    private readonly parser: GLTFParser;
+    private readonly _getDependency: any;
+    private readonly _loadingId: string;
+    private _loadedObjects: Set<object> = new Set();
+
+    constructor(parser: GLTFParser) {
+        this.parser = parser;
+        this._getDependency = this.parser.getDependency;
+        this._loadingId = Date.now().toString()
+    }
+
+    beforeRoot() {
+        InternalUsageTrackerPlugin._loadingProcesses++;
+        const self = this;
+        // Patch parser get dependency to track all objects that have been loaded or created
+        const getDependency = this._getDependency;
+        this.parser.getDependency = function (type: string, index: number) {
+            const promise = getDependency.call(this, type, index);
+            promise.then((result) => {
+                if (result) {
+                    self._loadedObjects.add(result);
+                    result[$loadingId] = self._loadingId;
+                }
+                return result;
+            });
+            return promise;
+        };
+        return null;
+    }
+
+    afterRoot(_result: GLTF) {
+        InternalUsageTrackerPlugin._loadingProcesses--;
+        // reset original method
+        this.parser.getDependency = this._getDependency;
+
+        // Cleanup usage of objects that have not been used in a scene
+        for (const loaded of this._loadedObjects) {
+            delete loaded[$loadingId];
+
+            if (loaded instanceof Object3D) {
+                if (!loaded.parent) {
+                    if (loaded instanceof Mesh) {
+                        if (debug) console.warn("> GLTF LOADER: Mesh not used in scene!", loaded);
+                        loaded.material = null;
+                        loaded.geometry = null;
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+
+
+
+
+    // private readonly _creatingNodeMesh: Map<number, CreateNodeMesh> = new Map();
+
+    // createNodeMesh(_nodeIndex: number): CreateNodeMesh | null {
+    //     // if (!this.parser) return null;
+    //     // let process = this._creatingNodeMesh.get(nodeIndex);
+    //     // if (process) return process;
+
+    //     // process = this.parser.createNodeMesh(nodeIndex)?.then((mesh) => {
+    //     //     console.log("createNodeMesh", nodeIndex, mesh);
+    //     //     return mesh;
+    //     // }) as CreateNodeMesh;
+    //     // this._creatingNodeMesh.set(nodeIndex, process);
+    //     // return process;
+    // }
+
+
+}
+

package/src/engine-components/AnimatorController.ts

@@ -768,6 +768,7 @@ class RootMotionAction {
     }
 
     onAfterUpdate() {
+        if (!this.action) return;
         const weight = this.action.getEffectiveWeight();
         this.positionChange.multiplyScalar(weight);
         this.rotationChange.slerp(RootMotionAction.identityQuaternion, 1 - weight);
@@ -836,6 +837,7 @@ class RootMotionHandler {
 
     private findRootTrack(clip: AnimationClip, name: string) {
         const tracks = clip.tracks;
+        if (!tracks) return null;
         for (const track of tracks) {
             if (track.name.endsWith(name)) {
                 // if (track.name.includes("Hips"))

package/src/engine-components/AudioSource.ts

@@ -65,7 +65,7 @@ export class AudioSource extends Behaviour {
             if (fn == undefined) return;
             if (AudioSource._userInteractionRegistered) return;
             AudioSource._userInteractionRegistered = true;
-            console.log("registered interaction, can play audio now");
+            if(debug) console.log("🔊 registered interaction, can play audio now");
             document.removeEventListener('pointerdown', fn);
             document.removeEventListener('click', fn);
             document.removeEventListener('dragstart', fn);

package/src/engine-components/Camera.ts

@@ -5,11 +5,12 @@ import { RGBAColor } from "./js-extensions/RGBAColor";
 import { Context, XRSessionMode } from "../engine/engine_setup";
 import { ICamera } from "../engine/engine_types"
 import { showBalloonMessage } from "../engine/debug/debug";
-import { getWorldPosition } from "../engine/engine_three_utils";
+import { getWorldPosition, Graphics } from "../engine/engine_three_utils";
 import { Gizmos } from "../engine/engine_gizmos";
 
 import { EquirectangularReflectionMapping, OrthographicCamera, PerspectiveCamera, Ray, sRGBEncoding, Vector3 } from "three";
 import { OrbitControls } from "./OrbitControls";
+import { RenderTexture } from "../engine/engine_texture";
 
 export enum ClearFlags {
     Skybox = 1,
@@ -151,6 +152,15 @@ export class Camera extends Behaviour implements ICamera {
         this.applyClearFlagsIfIsActiveCamera();
     }
 
+    @serializable(RenderTexture)
+    public set targetTexture(rt: RenderTexture | null) {
+        this._targetTexture = rt;
+    }
+    public get targetTexture(): RenderTexture | null {
+        return this._targetTexture;
+    }
+    private _targetTexture: RenderTexture | null = null;
+
     private _backgroundColor?: RGBAColor;
     private _fov?: number;
     private _cam: THREE.PerspectiveCamera | THREE.OrthographicCamera | null = null;
@@ -217,6 +227,28 @@ export class Camera extends Behaviour implements ICamera {
         this.context.removeCamera(this);
     }
 
+    onBeforeRender() {
+        if (this._cam) {
+            if (this._targetTexture) {
+                if (this.context.isManagedExternally) {
+                    // TODO: rendering with r3f renderer does throw an shader error for some reason?
+                    if (!this["_warnedAboutExternalRenderer"]) {
+                        this["_warnedAboutExternalRenderer"] = true;
+                        console.warn("Rendering with external renderer is not supported yet. This may not work or throw errors. Please remove the the target texture from your camera: " + this.name, this.targetTexture)
+                    }
+                }
+
+                // TODO: optimize to not render twice if this is already the main camera. In that case we just want to blit
+                const composer = this.context.composer;
+                const useNormalRenderer = true;// this.context.isInXR || !composer;
+                const renderer = useNormalRenderer ? this.context.renderer : composer;
+                if (renderer) {
+                    this._targetTexture.render(this.context.scene, this._cam, renderer)
+                }
+            }
+        }
+    }
+
     buildCamera() {
         if (this._cam) return;
 
@@ -262,43 +294,51 @@ export class Camera extends Behaviour implements ICamera {
     }
 
     applyClearFlagsIfIsActiveCamera() {
+        if (this.context.mainCameraComponent === this) {
+            this.applyClearFlags();
+        }
+    }
+
+    applyClearFlags() {
+        if (!this._cam) {
+            if (debug) console.log("Camera does not exist (apply clear flags)")
+            return;
+        }
         if (debug)
-            showBalloonMessage("apply Camera clear flags");
-        if (this._cam && this.context.mainCameraComponent === this) {
-            switch (this._clearFlags) {
-                case ClearFlags.Skybox:
-                    if (Camera.backgroundShouldBeTransparent(this.context)) {
-                        if (!this.ARBackgroundAlpha || this.ARBackgroundAlpha < 0.001) {
-                            this.context.scene.background = null;
-                            this.context.renderer.setClearColor(0x000000, 0);
-                            return;
-                        }
-                    }
-                    this.enableSkybox();
-
-                    if (this._backgroundBlurriness !== undefined)
-                        this.context.scene.backgroundBlurriness = this._backgroundBlurriness;
-                    if (this._backgroundIntensity !== undefined)
-                        //@ts-ignore
-                        this.context.scene.backgroundIntensity = this._backgroundIntensity;
-
-                    break;
-                case ClearFlags.SolidColor:
-                    if (this._backgroundColor) {
-                        let alpha = this._backgroundColor.alpha;
-                        // when in WebXR use ar background alpha override or set to 0
-                        if (Camera.backgroundShouldBeTransparent(this.context)) {
-                            alpha = this.ARBackgroundAlpha ?? 0;
-                        }
+            showBalloonMessage("apply Camera clear flags: " + this._clearFlags);
+        switch (this._clearFlags) {
+            case ClearFlags.Skybox:
+                if (Camera.backgroundShouldBeTransparent(this.context)) {
+                    if (!this.ARBackgroundAlpha || this.ARBackgroundAlpha < 0.001) {
                         this.context.scene.background = null;
-                        this.context.renderer.setClearColor(this._backgroundColor, alpha);
+                        this.context.renderer.setClearColor(0x000000, 0);
+                        return;
                     }
-                    break;
-                case ClearFlags.Uninitialized:
-                    this.context.scene.background = null
-                    this.context.renderer.setClearColor(0x000000, 0);
-                    break;
-            }
+                }
+                this.enableSkybox();
+
+                if (this._backgroundBlurriness !== undefined)
+                    this.context.scene.backgroundBlurriness = this._backgroundBlurriness;
+                if (this._backgroundIntensity !== undefined)
+                    //@ts-ignore
+                    this.context.scene.backgroundIntensity = this._backgroundIntensity;
+
+                break;
+            case ClearFlags.SolidColor:
+                if (this._backgroundColor) {
+                    let alpha = this._backgroundColor.alpha;
+                    // when in WebXR use ar background alpha override or set to 0
+                    if (Camera.backgroundShouldBeTransparent(this.context)) {
+                        alpha = this.ARBackgroundAlpha ?? 0;
+                    }
+                    this.context.scene.background = null;
+                    this.context.renderer.setClearColor(this._backgroundColor, alpha);
+                }
+                break;
+            case ClearFlags.Uninitialized:
+                this.context.scene.background = null
+                this.context.renderer.setClearColor(0x000000, 0);
+                break;
         }
     }
 
@@ -309,7 +349,7 @@ export class Camera extends Behaviour implements ICamera {
     }
 
     /** Returns true when in XR on a pass through device where the background shouldbe invisible */
-    static backgroundShouldBeTransparent(context:Context) {
+    static backgroundShouldBeTransparent(context: Context) {
         const session = context.renderer.xr?.getSession();
         if (!session) return false;
         const environmentBlendMode = session.environmentBlendMode;
@@ -330,7 +370,7 @@ export class Camera extends Behaviour implements ICamera {
                 }
             }
         }
-        
+
         return transparent;
     }
 }