@needle-tools/engine 4.5.0-alpha.1 → 4.5.0-alpha.3

package/src/engine/engine_loaders.ts

@@ -1,82 +1,360 @@
+import { BufferGeometry, Cache, Camera, Color, Loader, Material, Mesh, MeshStandardMaterial, Object3D } from "three";
+import { FBXLoader } from 'three/examples/jsm/loaders/FBXLoader.js'
+import { type GLTF, GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
+import { OBJLoader } from 'three/examples/jsm/loaders/OBJLoader.js';
+import { USDZLoader } from 'three/examples/jsm/loaders/USDZLoader.js';
 
-import { addDracoAndKTX2Loaders as addDracoAndKTX2LoadersGLTFProgressive, configureLoader, createLoaders, setDracoDecoderLocation, setKTX2TranscoderLocation } from '@needle-tools/gltf-progressive';
-import { MeshoptDecoder } from 'three/examples/jsm/libs/meshopt_decoder.module.js';
-import { DRACOLoader } from 'three/examples/jsm/loaders/DRACOLoader.js';
-import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
-import { KTX2Loader } from 'three/examples/jsm/loaders/KTX2Loader.js';
-
+import { showBalloonMessage } from "./debug/index.js";
+import { getLoader, type INeedleGltfLoader, registerLoader } from "./engine_gltf.js";
+import { createBuiltinComponents, writeBuiltinComponentData } from "./engine_gltf_builtin_components.js";
+import { CustomLoader, registeredModelLoaderCallbacks, ValidLoaderReturnType } from "./engine_loaders.callbacks.js";
+import * as loaders from "./engine_loaders.gltf.js"
+import { registerPrewarmObject } from "./engine_mainloop_utils.js";
+import { SerializationContext } from "./engine_serialization_core.js";
 import { Context } from "./engine_setup.js"
-import { getParam } from "./engine_utils.js";
+import { postprocessFBXMaterials } from "./engine_three_utils.js";
+import { CustomModel, isGLTFModel, Model, type UIDProvider } from "./engine_types.js";
+import * as utils from "./engine_utils.js";
+import { tryDetermineFileTypeFromURL } from "./engine_utils_format.js"
+import { invokeLoadedImportPluginHooks, registerComponentExtension, registerExtensions } from "./extensions/extensions.js";
+import { NEEDLE_components } from "./extensions/NEEDLE_components.js";
+
+/** @internal */
+export class NeedleLoader implements INeedleGltfLoader {
+    createBuiltinComponents(context: Context, gltfId: string, gltf: any, seed: number | UIDProvider | null, extension?: NEEDLE_components | undefined) {
+        return createBuiltinComponents(context, gltfId, gltf, seed, extension);
+    }
+    writeBuiltinComponentData(comp: any, context: SerializationContext) {
+        return writeBuiltinComponentData(comp, context);
+    }
+    parseSync(context: Context, data: string | ArrayBuffer, path: string, seed: number | UIDProvider | null): Promise<Model | undefined> {
+        return parseSync(context, data, path, seed);
+    }
+    loadSync(context: Context, url: string, sourceId: string, seed: number | UIDProvider | null, prog?: ((ProgressEvent: any) => void) | undefined): Promise<Model | undefined> {
+        return loadSync(context, url, sourceId, seed, prog);
+    }
+}
+registerLoader(NeedleLoader); // Register the loader
+
+
+const printGltf = utils.getParam("printGltf") || utils.getParam("printgltf");
+const debugFileTypes = utils.getParam("debugfileformat");
+
 
-const debug = getParam("debugdecoders");
 
-let loaders: null | { dracoLoader: DRACOLoader, ktx2Loader: KTX2Loader, meshoptDecoder: typeof MeshoptDecoder } = null;
 
-function ensureLoaders() {
-    if (!loaders) {
-        const res = createLoaders(null);
-        loaders = { dracoLoader: res.dracoLoader, ktx2Loader: res.ktx2Loader, meshoptDecoder: res.meshoptDecoder };
+export async function onCreateLoader(url: string, context: Context): Promise<CustomLoader | GLTFLoader | FBXLoader | USDZLoader | OBJLoader | null> {
+
+    const type = await tryDetermineFileTypeFromURL(url, { useExtension: true }) || "unknown";
+    if (debugFileTypes) console.debug(`Determined file type: '${type}' for url '${url}'`);
+
+    for (const callback of registeredModelLoaderCallbacks) {
+        const loader = callback({ context, url, filetype: type });
+        if (loader) {
+            console.debug("Using custom loader");
+            return loader;
+        }
     }
-    return loaders;
-}
 
-export function setDracoDecoderPath(path: string | undefined) {
-    if (path !== undefined && typeof path === "string") {
-        setDracoDecoderLocation(path);
+    switch (type) {
+        default:
+            console.warn(`Unknown file type: ${type}`);
+        case "unknown":
+            {
+                console.warn("Unknown file type. Assuming glTF:", url);
+                const loader = new GLTFLoader();
+                await registerExtensions(loader, context, url);
+                return loader;
+            }
+        case "fbx":
+            return new FBXLoader();
+        case "obj":
+            return new OBJLoader();
+        case "usd":
+        case "usda":
+        case "usdz":
+            {
+                console.warn(type.toUpperCase() + " files are not supported.");
+                // return new USDZLoader();
+                return null;
+            }
+        case "gltf":
+        case "glb":
+        case "vrm":
+            {
+                const loader = new GLTFLoader();
+                await registerExtensions(loader, context, url);
+                return loader;
+            }
     }
 }
 
-export function setDracoDecoderType(type: string | undefined) {
-    if (type !== undefined && typeof type === "string") {
-        if (type !== "js") {
-            const loaders = ensureLoaders();
-            if (debug) console.log("Setting draco decoder type to", type);
-            loaders.dracoLoader.setDecoderConfig({ type: type });
-        }
-    }
+/**
+ * Load a 3D model file from a remote URL
+ * @param url URL to glTF, FBX or OBJ file
+ * @param options
+ * @returns A promise that resolves to the loaded model or undefined if the loading failed
+ */
+export function loadAsset(url: string, options?: { context?: Context, path?: string, seed?: number, onprogress?: (evt: ProgressEvent) => void }): Promise<Model | undefined> {
+    return loadSync(options?.context || Context.Current, url, url, options?.seed || null, options?.onprogress);
 }
 
-export function setKtx2TranscoderPath(path: string) {
-    if (path !== undefined && typeof path === "string") {
-        setKTX2TranscoderLocation(path);
+/** Load a gltf file from a url. This is the core method used by Needle Engine to load gltf files. All known extensions are registered here.   
+ * @param context The current context
+ * @param data The gltf data as string or ArrayBuffer
+ * @param path The path to the gltf file
+ * @param seed The seed for generating unique ids
+ * @returns The loaded gltf object
+ */
+export async function parseSync(context: Context, data: string | ArrayBuffer, path: string, seed: number | UIDProvider | null): Promise<Model | undefined> {
+    if (typeof path !== "string") {
+        console.warn("Parse gltf binary without path, this might lead to errors in resolving extensions. Please provide the source path of the gltf/glb file", path, typeof path);
+        path = "";
     }
-}
+    if (printGltf) console.log("Parse glTF", path)
+    const loader = await onCreateLoader(path, context);
+    if (!loader) {
+        return undefined;
+    }
+
+    const { componentsExtension } = onBeforeLoad(loader, context);
 
-export function setMeshoptDecoder(_meshoptDecoder: any) {
-    if (_meshoptDecoder !== undefined) {
-        const loaders = ensureLoaders();
-        loaders.meshoptDecoder = _meshoptDecoder;
+    // Handle OBJ Loader
+    if (loader instanceof OBJLoader) {
+        if (typeof data !== "string") { data = new TextDecoder().decode(data); }
+        const res = loader.parse(data);
+        return await onAfterLoaded(loader, context, path, res, seed, componentsExtension);
+    }
+    // Handle any other loader that is not a GLTFLoader
+    const isNotGLTF = !(loader instanceof GLTFLoader);
+    if (isNotGLTF) {
+        if (loader.parse === undefined) {
+            console.error("Loader does not support parse");
+            return undefined;
+        }
+        const res = loader.parse(data, path);
+        return await onAfterLoaded(loader, context, path, res, seed, componentsExtension);
     }
+
+    return new Promise((resolve, reject) => {
+        try {
+
+            // GltfLoader expects a base path for resolving referenced assets
+            // https://threejs.org/docs/#examples/en/loaders/GLTFLoader.parse
+            // so we make sure that "path" is never a file path
+            let gltfLoaderPath = path.split("?")[0].trimEnd();
+            // This assumes that the path is a FILE path and not already a directory
+            // (it does not end with "/") – see https://linear.app/needle/issue/NE-6075
+            // strip file from path
+            const parts = gltfLoaderPath.split("/");
+            // check if the last part is a /, otherwise remove it
+            if (parts.length > 0 && parts[parts.length - 1] !== "")
+                parts.pop();
+            gltfLoaderPath = parts.join("/");
+            if (!gltfLoaderPath.endsWith("/")) gltfLoaderPath += "/";
+
+            loader.resourcePath = gltfLoaderPath;
+            loader.parse(data, "", async res => {
+                const model = await onAfterLoaded(loader, context, path, res, seed, componentsExtension);
+                resolve(model);
+
+            }, err => {
+                console.error("Loading asset at \"" + path + "\" failed\n", err);
+                resolve(undefined);
+            });
+        }
+        catch (err) {
+            console.error(err);
+            reject(err);
+        }
+    });
 }
 
 /**
- * Add Draco, Meshopt and KTX2 loaders to a GLTFLoader instance.
- * @param loader The GLTFLoader instance to add the loaders to.
- * @param context The context object containing the renderer.
- * @returns The GLTFLoader instance with the loaders added.
+ * Load a gltf file from a url. This is the core method used by Needle Engine to load gltf files. All known extensions are registered here.  
+ * @param context The current context
+ * @param url The url to the gltf file
+ * @param sourceId The source id of the gltf file - this is usually the url
+ * @param seed The seed for generating unique ids
+ * @param prog A progress callback
+ * @returns The loaded gltf object
  */
-export function addDracoAndKTX2Loaders(loader: GLTFLoader, context: Pick<Context, "renderer">) {
+export async function loadSync(context: Context, url: string, sourceId: string, seed: number | UIDProvider | null, prog?: (ProgressEvent) => void): Promise<Model | undefined> {
 
-    const loaders = ensureLoaders();
+    checkIfUserAttemptedToLoadALocalFile(url);
 
-    if (context.renderer) {
-        loaders.ktx2Loader.detectSupport(context.renderer);
+    // better to create new loaders every time
+    // (maybe we can cache them...)
+    // but due to the async nature and potentially triggering multiple loads at the same time
+    // we need to make sure the extensions dont override each other
+    // creating new loaders should not be expensive as well
+    const loader = await onCreateLoader(url, context);
+    if (!loader) {
+        return undefined;
     }
-    else
-        console.warn("No renderer provided to detect ktx2 support - loading KTX2 textures will probably fail");
 
-    addDracoAndKTX2LoadersGLTFProgressive(loader);
+    const { componentsExtension } = onBeforeLoad(loader, context);
 
-    if (!loader.dracoLoader)
-        loader.setDRACOLoader(loaders.dracoLoader);
-    if (!(loader as any).ktx2Loader)
-        loader.setKTX2Loader(loaders.ktx2Loader);
-    if (!(loader as any).meshoptDecoder)
-        loader.setMeshoptDecoder(loaders.meshoptDecoder);
+    // Handle any loader that is not a GLTFLoader
+    if (!(loader instanceof GLTFLoader)) {
+        const res = await loader.loadAsync(url, prog);
+        return await onAfterLoaded(loader, context, url, res, seed, componentsExtension);
+    }
 
-    configureLoader(loader, {
-        progressive: true,
+    return new Promise((resolve, reject) => {
+        try {
+            loader.load(url, async res => {
+                const model = await onAfterLoaded(loader, context, sourceId, res, seed, componentsExtension);
+                resolve(model);
+            }, evt => {
+                prog?.call(loader, evt);
+            }, err => {
+                console.error("Loading asset at \"" + url + "\" failed\n", err);
+                resolve(undefined);
+            });
+        }
+        catch (err) {
+            console.error(err);
+            reject(err);
+        }
     });
+}
+
+/** Call before loading a model */
+function onBeforeLoad(loader: Loader | CustomLoader, context: Context): { componentsExtension: NEEDLE_components | null } {
+    const componentsExtension = registerComponentExtension(loader);
+    if (loader instanceof GLTFLoader) {
+        loaders.addDracoAndKTX2Loaders(loader, context);
+    }
+    return { componentsExtension };
+}
 
-    return loader;
+
+/** Call after a 3d model has been loaded to compile shaders and construct the needle engine model structure with relevant metadata (if necessary) */
+async function onAfterLoaded(loader: Loader | CustomLoader, context: Context, gltfId: string, model: ValidLoaderReturnType, seed: number | null | UIDProvider, componentsExtension: NEEDLE_components | null): Promise<Model> {
+    if (printGltf) console.warn("Loaded", gltfId, model);
+
+    // Handle loader was registered but no model was returned - should not completely break the engine
+    if (model == null) {
+        console.error(`Loaded model is null '${gltfId}' - please make sure the loader is registered correctly`);
+        return {
+            scene: new Object3D(),
+            animations: [],
+            scenes: []
+        };
+    }
+
+    // Handle OBJ or FBX loader results
+    if (model instanceof Object3D) {
+        model = {
+            scene: model,
+            animations: model.animations,
+            scenes: [model]
+        }
+    }
+    // Handle STL loader results
+    else if (model instanceof BufferGeometry) {
+        const mat = new MeshStandardMaterial({
+            color: new Color(0xdddddd)
+        });
+        const mesh = new Mesh(model, mat);
+        model = {
+            scene: mesh,
+            animations: [],
+            scenes: [mesh]
+        }
+    }
+
+
+    // Remove query parameters from gltfId
+    if (gltfId.includes("?")) {
+        gltfId = gltfId.split("?")[0];
+    }
+
+    // assign animations of loaded glTF to all scenes
+    for (const scene of model.scenes) {
+        if (scene && !scene.animations?.length) {
+            scene.animations = [...model.animations];
+        }
+    }
+
+    // E.g. fbx material cleanup
+    postprocessLoadedFile(loader, model);
+
+    // load components
+    if (isGLTFModel(model)) {
+        invokeLoadedImportPluginHooks(gltfId, model, context);
+        await getLoader().createBuiltinComponents(context, gltfId, model, seed, componentsExtension || undefined);
+    }
+
+    // Warmup the scene
+    await compileAsync(model.scene, context, context.mainCamera);
+
+    return model;
 }
+
+async function compileAsync(scene: Object3D, context: Context, camera?: Camera | null) {
+    if (!camera) camera = context.mainCamera;
+    try {
+        if (camera) {
+            await context.renderer.compileAsync(scene, camera, context.scene)
+                .catch(err => {
+                    console.warn(err.message);
+                });
+        }
+        else
+            registerPrewarmObject(scene, context);
+    }
+    catch (err: Error | any) {
+        console.warn(err?.message || err);
+    }
+}
+
+function checkIfUserAttemptedToLoadALocalFile(url: string) {
+    const fullurl = new URL(url, window.location.href).href;
+    if (fullurl.startsWith("file://")) {
+        const msg = "Hi - it looks like you are trying to load a local file which will not work. You need to use a webserver to serve your files.\nPlease refer to the documentation on <a href=\"https://fwd.needle.tools/needle-engine/docs/local-server\">https://docs.needle.tools</a> or ask for help in our <a href=\"https://discord.needle.tools\">discord community</a>";
+        showBalloonMessage(msg);
+        console.warn(msg)
+    }
+}
+
+// function _downloadGltf(data: string | ArrayBuffer) {
+//     if (typeof data === "string") {
+//         const a = document.createElement("a") as HTMLAnchorElement;
+//         a.href = data;
+//         a.download = data.split("/").pop()!;
+//         a.click();
+//     }
+//     else {
+//         const blob = new Blob([data], { type: "application/octet-stream" });
+//         const url = window.URL.createObjectURL(blob);
+//         const a = document.createElement("a") as HTMLAnchorElement;
+//         a.href = url;
+//         a.download = "download.glb";
+//         a.click();
+//     }
+// }
+
+/**
+ * Postprocess the loaded file. This is used to apply any custom postprocessing to the loaded file.
+ */
+function postprocessLoadedFile(loader: object, result: Model) {
+
+    if (loader instanceof FBXLoader || loader instanceof OBJLoader) {
+
+        let obj: Object3D | Model = result;
+        if (!(obj instanceof Object3D)) {
+            obj = (result as GLTF).scene;
+        }
+
+        obj.traverse((child) => {
+            const mesh = child as Mesh;
+            // See https://github.com/needle-tools/three.js/blob/b8df3843ff123ac9dc0ed0d3ccc5b568f840c804/examples/webgl_loader_multiple.html#L377
+            if (mesh?.isMesh) {
+                postprocessFBXMaterials(mesh, mesh.material as Material);
+            }
+        });
+    }
+}
+

package/src/engine/engine_types.ts

@@ -1,5 +1,5 @@
 import type { QueryFilterFlags, World } from "@dimforge/rapier3d-compat";
-import { AnimationClip, Color, Material, Mesh, Object3D, Quaternion } from "three";
+import { AnimationClip, Color, Loader, Material, Mesh, Object3D, Quaternion } from "three";
 import { Vector3 } from "three";
 import { type GLTF as THREE_GLTF } from "three/examples/jsm/loaders/GLTFLoader.js";
 
@@ -21,8 +21,10 @@ type NoInternals<T> = FilterStartingWith<T, "_">;
 type NoInternalNeedleEngineState<T> = Omit<T, "destroyed" | "gameObject" | "activeAndEnabled" | "context" | "isComponent" | "scene" | "up" | "forward" | "right" | "worldRotation" | "worldEuler" | "worldPosition" | "worldQuaternion">;
 export type ComponentInit<T> = Partial<NoInternalNeedleEngineState<NoInternals<NoUndefinedNoFunctions<T>>>>
 
+
 /** GLTF, GLB or VRM */
 export type GLTF = THREE_GLTF;
+
 /** FBX type */
 export type FBX = {
     animations: AnimationClip[];
@@ -35,8 +37,22 @@ export type OBJ = {
     scene: Object3D;
     scenes: Object3D[];
 }
+export type CustomModel = {
+    animations: AnimationClip[];
+    scene: Object3D;
+    scenes: Object3D[];
+}
+
 /** All possible model types that Needle Engine can load */
-export type Model = (GLTF | FBX | OBJ);
+export type Model = (GLTF | FBX | OBJ | CustomModel);
+
+export function isGLTFModel(model: Model): model is GLTF {
+    const gltf = model as GLTF;
+    if (gltf.parser && gltf.parser.json) {
+        return true;
+    }
+    return false;
+}
 
 /** A loaded model */
 export type LoadedModel = {
@@ -218,7 +234,7 @@ export interface IComponent extends IHasGuid {
     get worldQuaternion(): Quaternion;
 }
 
-export function isComponent(obj:any) : obj is IComponent {
+export function isComponent(obj: any): obj is IComponent {
     return obj && obj.isComponent;
 }
 
@@ -547,14 +563,14 @@ export interface IPhysicsEngine {
      * @param collider The collider component to add
      */
     addSphereCollider(collider: ICollider);
-    
+
     /**
      * Adds a box collider to the physics world
      * @param collider The collider component to add
      * @param size The size of the box
      */
     addBoxCollider(collider: ICollider, size: Vector3);
-    
+
     /**
      * Adds a capsule collider to the physics world
      * @param collider The collider component to add
@@ -562,7 +578,7 @@ export interface IPhysicsEngine {
      * @param height The height of the capsule
      */
     addCapsuleCollider(collider: ICollider, radius: number, height: number);
-    
+
     /**
      * Adds a mesh collider to the physics world
      * @param collider The collider component to add
@@ -584,34 +600,34 @@ export interface IPhysicsEngine {
      * @param rb The rigidbody to wake up
      */
     wakeup(rb: IRigidbody);
-    
+
     /**
      * Checks if a rigidbody is currently sleeping
      * @param rb The rigidbody to check
      * @returns Whether the rigidbody is sleeping or undefined if cannot be determined
      */
     isSleeping(rb: IRigidbody): boolean | undefined;
-    
+
     /**
      * Updates the physical properties of a rigidbody or collider
      * @param rb The rigidbody or collider to update
      */
     updateProperties(rb: IRigidbody | ICollider);
-    
+
     /**
      * Resets all forces acting on a rigidbody
      * @param rb The rigidbody to reset forces on
      * @param wakeup Whether to wake up the rigidbody
      */
     resetForces(rb: IRigidbody, wakeup: boolean);
-    
+
     /**
      * Resets all torques acting on a rigidbody
      * @param rb The rigidbody to reset torques on
      * @param wakeup Whether to wake up the rigidbody
      */
     resetTorques(rb: IRigidbody, wakeup: boolean);
-    
+
     /**
      * Adds a continuous force to a rigidbody
      * @param rb The rigidbody to add force to
@@ -619,7 +635,7 @@ export interface IPhysicsEngine {
      * @param wakeup Whether to wake up the rigidbody
      */
     addForce(rb: IRigidbody, vec: Vec3, wakeup: boolean);
-    
+
     /**
      * Applies an instantaneous impulse to a rigidbody
      * @param rb The rigidbody to apply impulse to
@@ -627,21 +643,21 @@ export interface IPhysicsEngine {
      * @param wakeup Whether to wake up the rigidbody
      */
     applyImpulse(rb: IRigidbody, vec: Vec3, wakeup: boolean);
-    
+
     /**
      * Gets the linear velocity of a rigidbody or the rigidbody attached to a collider
      * @param rb The rigidbody or collider to get velocity from
      * @returns The linear velocity vector or null if not available
      */
     getLinearVelocity(rb: IRigidbody | ICollider): Vec3 | null;
-    
+
     /**
      * Gets the angular velocity of a rigidbody
      * @param rb The rigidbody to get angular velocity from
      * @returns The angular velocity vector or null if not available
      */
     getAngularVelocity(rb: IRigidbody): Vec3 | null;
-    
+
     /**
      * Sets the angular velocity of a rigidbody
      * @param rb The rigidbody to set angular velocity for
@@ -649,7 +665,7 @@ export interface IPhysicsEngine {
      * @param wakeup Whether to wake up the rigidbody
      */
     setAngularVelocity(rb: IRigidbody, vec: Vec3, wakeup: boolean);
-    
+
     /**
      * Sets the linear velocity of a rigidbody
      * @param rb The rigidbody to set linear velocity for
@@ -665,13 +681,13 @@ export interface IPhysicsEngine {
      * @param rotation Whether to update the rotation
      */
     updateBody(comp: ICollider | IRigidbody, translation: boolean, rotation: boolean);
-    
+
     /**
      * Removes a physics body from the simulation
      * @param body The component whose physics body should be removed
      */
     removeBody(body: IComponent);
-    
+
     /**
      * Gets the physics body for a component
      * @param obj The collider or rigidbody component