@needle-tools/engine 4.5.0-alpha.1 → 4.5.0-alpha.2

package/src/engine/engine_loaders.callbacks.ts

@@ -0,0 +1,88 @@
+import { BufferGeometry, Object3D } from "three";
+import { Context } from "./engine_setup.js";
+import { CustomModel } from "./engine_types.js";
+import { type FileType } from "./engine_utils_format.js";
+
+export type ValidLoaderReturnType = CustomModel | Object3D | BufferGeometry;
+
+
+/** @internal */
+export type CustomLoader = {
+    /** The name of the loader. This is used for debugging purposes. */
+    name?: string,
+    /** Load the model from the given URL. This method should return a promise that resolves to the loaded model. */
+    loadAsync: (url: string, onProgress?: ((event: ProgressEvent<EventTarget>) => void) | undefined) => Promise<ValidLoaderReturnType>,
+    /** Load the model given a buffer. This method should return the loaded model. */
+    parse: (buffer: ArrayBuffer | string, path: string) => ValidLoaderReturnType,
+};
+
+type CustomLoaderCallback = (args: { context: Context, url: string, filetype: FileType }) => CustomLoader | null | undefined | void;
+/** @internal */
+export const registeredModelLoaderCallbacks: Array<CustomLoaderCallback> = [];
+
+
+type FileTypeCallback = (args: {
+    /** The URL of the file to load */
+    url: string,
+    /** The response of the range request with the first few bytes of the file (bytes are available in the 'args.bytes' property of this callback) */
+    response: Response,
+    /** The mimetype of the file as provided by the request header */
+    contentType: string | null,
+    /** The first few bytes of the file as a Uint8Array */
+    bytes: Uint8Array
+}) => FileType | null;
+/** @internal */
+export const registeredFileTypeCallbacks: Array<FileTypeCallback> = [];
+
+
+/**
+ * NeedleEngineModelLoader is a namespace that provides functions to register custom model loaders and mimetype callbacks.
+ * It allows you to create custom loaders for specific file types and determine the mimetype of files based on their content.
+ */
+export namespace NeedleEngineModelLoader {
+
+    // export type ModelLoaderPlugin = {
+    //     name: string,
+    //     canLoad: (url: string, mimetype: string) => boolean,
+    // }
+    
+    /**
+     * Register a custom loader callback. For every file that is requested this callback is called with the url and mimetype. It should return a custom loader or null if it does not want to handle the file.
+     * @param callback The callback to register
+     * @returns A function to unregister the callback
+     * @example
+     * ```ts
+     * import { onCreateModelLoader } from "@needle-tools/engine";
+     * const unregister = onCreateModelLoader((url, mimetype) => {
+     *     if (mimetype === "application/vnd.usdz+zip") {
+     *         return new CustomLoader();
+     *     }
+     *     return null;
+     * });
+     */
+    export function onCreateCustomModelLoader(callback: CustomLoaderCallback) {
+        registeredModelLoaderCallbacks.push(callback);
+        return () => {
+            const index = registeredModelLoaderCallbacks.indexOf(callback);
+            if (index >= 0) {
+                registeredModelLoaderCallbacks.splice(index, 1);
+            }
+        }
+    }
+
+    /**
+     * Register a callback to determine the mimetype of a file. This is to support custom loaders. The callback will provide the URL of the file to load + a range request response with the first few bytes of the file. The callback should return a mimetype or null if it does not want to handle the file.
+     * @param callback The callback to register
+     * @returns A function to unregister the callback
+     * 
+     */
+    export function onDetermineModelFiletype(callback: FileTypeCallback): (() => void) {
+        registeredFileTypeCallbacks.push(callback);
+        return () => {
+            const index = registeredFileTypeCallbacks.indexOf(callback);
+            if (index >= 0) {
+                registeredFileTypeCallbacks.splice(index, 1);
+            }
+        }
+    }
+}

package/src/engine/engine_loaders.gltf.ts

@@ -0,0 +1,82 @@
+
+import { addDracoAndKTX2Loaders as addDracoAndKTX2LoadersGLTFProgressive, configureLoader, createLoaders, setDracoDecoderLocation, setKTX2TranscoderLocation } from '@needle-tools/gltf-progressive';
+import { MeshoptDecoder } from 'three/examples/jsm/libs/meshopt_decoder.module.js';
+import { DRACOLoader } from 'three/examples/jsm/loaders/DRACOLoader.js';
+import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
+import { KTX2Loader } from 'three/examples/jsm/loaders/KTX2Loader.js';
+
+import { Context } from "./engine_setup.js"
+import { getParam } from "./engine_utils.js";
+
+const debug = getParam("debugdecoders");
+
+let loaders: null | { dracoLoader: DRACOLoader, ktx2Loader: KTX2Loader, meshoptDecoder: typeof MeshoptDecoder } = null;
+
+function ensureLoaders() {
+    if (!loaders) {
+        const res = createLoaders(null);
+        loaders = { dracoLoader: res.dracoLoader, ktx2Loader: res.ktx2Loader, meshoptDecoder: res.meshoptDecoder };
+    }
+    return loaders;
+}
+
+export function setDracoDecoderPath(path: string | undefined) {
+    if (path !== undefined && typeof path === "string") {
+        setDracoDecoderLocation(path);
+    }
+}
+
+export function setDracoDecoderType(type: string | undefined) {
+    if (type !== undefined && typeof type === "string") {
+        if (type !== "js") {
+            const loaders = ensureLoaders();
+            if (debug) console.log("Setting draco decoder type to", type);
+            loaders.dracoLoader.setDecoderConfig({ type: type });
+        }
+    }
+}
+
+export function setKtx2TranscoderPath(path: string) {
+    if (path !== undefined && typeof path === "string") {
+        setKTX2TranscoderLocation(path);
+    }
+}
+
+export function setMeshoptDecoder(_meshoptDecoder: any) {
+    if (_meshoptDecoder !== undefined) {
+        const loaders = ensureLoaders();
+        loaders.meshoptDecoder = _meshoptDecoder;
+    }
+}
+
+/**
+ * Add Draco, Meshopt and KTX2 loaders to a GLTFLoader instance.
+ * @param loader The GLTFLoader instance to add the loaders to.
+ * @param context The context object containing the renderer.
+ * @returns The GLTFLoader instance with the loaders added.
+ */
+export function addDracoAndKTX2Loaders(loader: GLTFLoader, context: Pick<Context, "renderer">) {
+
+    const loaders = ensureLoaders();
+
+    if (context.renderer) {
+        loaders.ktx2Loader.detectSupport(context.renderer);
+    }
+    else
+        console.warn("No renderer provided to detect ktx2 support - loading KTX2 textures will probably fail");
+
+    addDracoAndKTX2LoadersGLTFProgressive(loader);
+
+    if (!loader.dracoLoader)
+        loader.setDRACOLoader(loaders.dracoLoader);
+    if (!(loader as any).ktx2Loader)
+        loader.setKTX2Loader(loaders.ktx2Loader);
+    if (!(loader as any).meshoptDecoder)
+        loader.setMeshoptDecoder(loaders.meshoptDecoder);
+
+    configureLoader(loader, {
+        progressive: true,
+    });
+
+    return loader;
+}

package/src/engine/engine_loaders.ts

@@ -1,82 +1,360 @@
+import { BufferGeometry, Cache, Camera, Color, Loader, Material, Mesh, MeshStandardMaterial, Object3D } from "three";
+import { FBXLoader } from 'three/examples/jsm/loaders/FBXLoader.js'
+import { type GLTF, GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
+import { OBJLoader } from 'three/examples/jsm/loaders/OBJLoader.js';
+import { USDZLoader } from 'three/examples/jsm/loaders/USDZLoader.js';
 
-import { addDracoAndKTX2Loaders as addDracoAndKTX2LoadersGLTFProgressive, configureLoader, createLoaders, setDracoDecoderLocation, setKTX2TranscoderLocation } from '@needle-tools/gltf-progressive';
-import { MeshoptDecoder } from 'three/examples/jsm/libs/meshopt_decoder.module.js';
-import { DRACOLoader } from 'three/examples/jsm/loaders/DRACOLoader.js';
-import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
-import { KTX2Loader } from 'three/examples/jsm/loaders/KTX2Loader.js';
-
+import { showBalloonMessage } from "./debug/index.js";
+import { getLoader, type INeedleGltfLoader, registerLoader } from "./engine_gltf.js";
+import { createBuiltinComponents, writeBuiltinComponentData } from "./engine_gltf_builtin_components.js";
+import { CustomLoader, registeredModelLoaderCallbacks, ValidLoaderReturnType } from "./engine_loaders.callbacks.js";
+import * as loaders from "./engine_loaders.gltf.js"
+import { registerPrewarmObject } from "./engine_mainloop_utils.js";
+import { SerializationContext } from "./engine_serialization_core.js";
 import { Context } from "./engine_setup.js"
-import { getParam } from "./engine_utils.js";
+import { postprocessFBXMaterials } from "./engine_three_utils.js";
+import { CustomModel, isGLTFModel, Model, type UIDProvider } from "./engine_types.js";
+import * as utils from "./engine_utils.js";
+import { tryDetermineFileTypeFromURL } from "./engine_utils_format.js"
+import { invokeLoadedImportPluginHooks, registerComponentExtension, registerExtensions } from "./extensions/extensions.js";
+import { NEEDLE_components } from "./extensions/NEEDLE_components.js";
+
+/** @internal */
+export class NeedleLoader implements INeedleGltfLoader {
+    createBuiltinComponents(context: Context, gltfId: string, gltf: any, seed: number | UIDProvider | null, extension?: NEEDLE_components | undefined) {
+        return createBuiltinComponents(context, gltfId, gltf, seed, extension);
+    }
+    writeBuiltinComponentData(comp: any, context: SerializationContext) {
+        return writeBuiltinComponentData(comp, context);
+    }
+    parseSync(context: Context, data: string | ArrayBuffer, path: string, seed: number | UIDProvider | null): Promise<Model | undefined> {
+        return parseSync(context, data, path, seed);
+    }
+    loadSync(context: Context, url: string, sourceId: string, seed: number | UIDProvider | null, prog?: ((ProgressEvent: any) => void) | undefined): Promise<Model | undefined> {
+        return loadSync(context, url, sourceId, seed, prog);
+    }
+}
+registerLoader(NeedleLoader); // Register the loader
+
+
+const printGltf = utils.getParam("printGltf") || utils.getParam("printgltf");
+const debugFileTypes = utils.getParam("debugfileformat");
+
 
-const debug = getParam("debugdecoders");
 
-let loaders: null | { dracoLoader: DRACOLoader, ktx2Loader: KTX2Loader, meshoptDecoder: typeof MeshoptDecoder } = null;
 
-function ensureLoaders() {
-    if (!loaders) {
-        const res = createLoaders(null);
-        loaders = { dracoLoader: res.dracoLoader, ktx2Loader: res.ktx2Loader, meshoptDecoder: res.meshoptDecoder };
+export async function onCreateLoader(url: string, context: Context): Promise<CustomLoader | GLTFLoader | FBXLoader | USDZLoader | OBJLoader | null> {
+
+    const type = await tryDetermineFileTypeFromURL(url, { useExtension: true }) || "unknown";
+    if (debugFileTypes) console.debug(`Determined file type: '${type}' for url '${url}'`);
+
+    for (const callback of registeredModelLoaderCallbacks) {
+        const loader = callback({ context, url, filetype: type });
+        if (loader) {
+            console.debug("Using custom loader");
+            return loader;
+        }
     }
-    return loaders;
-}
 
-export function setDracoDecoderPath(path: string | undefined) {
-    if (path !== undefined && typeof path === "string") {
-        setDracoDecoderLocation(path);
+    switch (type) {
+        default:
+            console.warn(`Unknown file type: ${type}`);
+        case "unknown":
+            {
+                console.warn("Unknown file type. Assuming glTF:", url);
+                const loader = new GLTFLoader();
+                await registerExtensions(loader, context, url);
+                return loader;
+            }
+        case "fbx":
+            return new FBXLoader();
+        case "obj":
+            return new OBJLoader();
+        case "usd":
+        case "usda":
+        case "usdz":
+            {
+                console.warn(type.toUpperCase() + " files are not supported.");
+                // return new USDZLoader();
+                return null;
+            }
+        case "gltf":
+        case "glb":
+        case "vrm":
+            {
+                const loader = new GLTFLoader();
+                await registerExtensions(loader, context, url);
+                return loader;
+            }
     }
 }
 
-export function setDracoDecoderType(type: string | undefined) {
-    if (type !== undefined && typeof type === "string") {
-        if (type !== "js") {
-            const loaders = ensureLoaders();
-            if (debug) console.log("Setting draco decoder type to", type);
-            loaders.dracoLoader.setDecoderConfig({ type: type });
-        }
-    }
+/**
+ * Load a 3D model file from a remote URL
+ * @param url URL to glTF, FBX or OBJ file
+ * @param options
+ * @returns A promise that resolves to the loaded model or undefined if the loading failed
+ */
+export function loadAsset(url: string, options?: { context?: Context, path?: string, seed?: number, onprogress?: (evt: ProgressEvent) => void }): Promise<Model | undefined> {
+    return loadSync(options?.context || Context.Current, url, url, options?.seed || null, options?.onprogress);
 }
 
-export function setKtx2TranscoderPath(path: string) {
-    if (path !== undefined && typeof path === "string") {
-        setKTX2TranscoderLocation(path);
+/** Load a gltf file from a url. This is the core method used by Needle Engine to load gltf files. All known extensions are registered here.   
+ * @param context The current context
+ * @param data The gltf data as string or ArrayBuffer
+ * @param path The path to the gltf file
+ * @param seed The seed for generating unique ids
+ * @returns The loaded gltf object
+ */
+export async function parseSync(context: Context, data: string | ArrayBuffer, path: string, seed: number | UIDProvider | null): Promise<Model | undefined> {
+    if (typeof path !== "string") {
+        console.warn("Parse gltf binary without path, this might lead to errors in resolving extensions. Please provide the source path of the gltf/glb file", path, typeof path);
+        path = "";
     }
-}
+    if (printGltf) console.log("Parse glTF", path)
+    const loader = await onCreateLoader(path, context);
+    if (!loader) {
+        return undefined;
+    }
+
+    const { componentsExtension } = onBeforeLoad(loader, context);
 
-export function setMeshoptDecoder(_meshoptDecoder: any) {
-    if (_meshoptDecoder !== undefined) {
-        const loaders = ensureLoaders();
-        loaders.meshoptDecoder = _meshoptDecoder;
+    // Handle OBJ Loader
+    if (loader instanceof OBJLoader) {
+        if (typeof data !== "string") { data = new TextDecoder().decode(data); }
+        const res = loader.parse(data);
+        return await onAfterLoaded(loader, context, path, res, seed, componentsExtension);
+    }
+    // Handle any other loader that is not a GLTFLoader
+    const isNotGLTF = !(loader instanceof GLTFLoader);
+    if (isNotGLTF) {
+        if (loader.parse === undefined) {
+            console.error("Loader does not support parse");
+            return undefined;
+        }
+        const res = loader.parse(data, path);
+        return await onAfterLoaded(loader, context, path, res, seed, componentsExtension);
     }
+
+    return new Promise((resolve, reject) => {
+        try {
+
+            // GltfLoader expects a base path for resolving referenced assets
+            // https://threejs.org/docs/#examples/en/loaders/GLTFLoader.parse
+            // so we make sure that "path" is never a file path
+            let gltfLoaderPath = path.split("?")[0].trimEnd();
+            // This assumes that the path is a FILE path and not already a directory
+            // (it does not end with "/") – see https://linear.app/needle/issue/NE-6075
+            // strip file from path
+            const parts = gltfLoaderPath.split("/");
+            // check if the last part is a /, otherwise remove it
+            if (parts.length > 0 && parts[parts.length - 1] !== "")
+                parts.pop();
+            gltfLoaderPath = parts.join("/");
+            if (!gltfLoaderPath.endsWith("/")) gltfLoaderPath += "/";
+
+            loader.resourcePath = gltfLoaderPath;
+            loader.parse(data, "", async res => {
+                const model = await onAfterLoaded(loader, context, path, res, seed, componentsExtension);
+                resolve(model);
+
+            }, err => {
+                console.error("Loading asset at \"" + path + "\" failed\n", err);
+                resolve(undefined);
+            });
+        }
+        catch (err) {
+            console.error(err);
+            reject(err);
+        }
+    });
 }
 
 /**
- * Add Draco, Meshopt and KTX2 loaders to a GLTFLoader instance.
- * @param loader The GLTFLoader instance to add the loaders to.
- * @param context The context object containing the renderer.
- * @returns The GLTFLoader instance with the loaders added.
+ * Load a gltf file from a url. This is the core method used by Needle Engine to load gltf files. All known extensions are registered here.  
+ * @param context The current context
+ * @param url The url to the gltf file
+ * @param sourceId The source id of the gltf file - this is usually the url
+ * @param seed The seed for generating unique ids
+ * @param prog A progress callback
+ * @returns The loaded gltf object
  */
-export function addDracoAndKTX2Loaders(loader: GLTFLoader, context: Pick<Context, "renderer">) {
+export async function loadSync(context: Context, url: string, sourceId: string, seed: number | UIDProvider | null, prog?: (ProgressEvent) => void): Promise<Model | undefined> {
 
-    const loaders = ensureLoaders();
+    checkIfUserAttemptedToLoadALocalFile(url);
 
-    if (context.renderer) {
-        loaders.ktx2Loader.detectSupport(context.renderer);
+    // better to create new loaders every time
+    // (maybe we can cache them...)
+    // but due to the async nature and potentially triggering multiple loads at the same time
+    // we need to make sure the extensions dont override each other
+    // creating new loaders should not be expensive as well
+    const loader = await onCreateLoader(url, context);
+    if (!loader) {
+        return undefined;
     }
-    else
-        console.warn("No renderer provided to detect ktx2 support - loading KTX2 textures will probably fail");
 
-    addDracoAndKTX2LoadersGLTFProgressive(loader);
+    const { componentsExtension } = onBeforeLoad(loader, context);
 
-    if (!loader.dracoLoader)
-        loader.setDRACOLoader(loaders.dracoLoader);
-    if (!(loader as any).ktx2Loader)
-        loader.setKTX2Loader(loaders.ktx2Loader);
-    if (!(loader as any).meshoptDecoder)
-        loader.setMeshoptDecoder(loaders.meshoptDecoder);
+    // Handle any loader that is not a GLTFLoader
+    if (!(loader instanceof GLTFLoader)) {
+        const res = await loader.loadAsync(url, prog);
+        return await onAfterLoaded(loader, context, url, res, seed, componentsExtension);
+    }
 
-    configureLoader(loader, {
-        progressive: true,
+    return new Promise((resolve, reject) => {
+        try {
+            loader.load(url, async res => {
+                const model = await onAfterLoaded(loader, context, sourceId, res, seed, componentsExtension);
+                resolve(model);
+            }, evt => {
+                prog?.call(loader, evt);
+            }, err => {
+                console.error("Loading asset at \"" + url + "\" failed\n", err);
+                resolve(undefined);
+            });
+        }
+        catch (err) {
+            console.error(err);
+            reject(err);
+        }
     });
+}
+
+/** Call before loading a model */
+function onBeforeLoad(loader: Loader | CustomLoader, context: Context): { componentsExtension: NEEDLE_components | null } {
+    const componentsExtension = registerComponentExtension(loader);
+    if (loader instanceof GLTFLoader) {
+        loaders.addDracoAndKTX2Loaders(loader, context);
+    }
+    return { componentsExtension };
+}
 
-    return loader;
+
+/** Call after a 3d model has been loaded to compile shaders and construct the needle engine model structure with relevant metadata (if necessary) */
+async function onAfterLoaded(loader: Loader | CustomLoader, context: Context, gltfId: string, model: ValidLoaderReturnType, seed: number | null | UIDProvider, componentsExtension: NEEDLE_components | null): Promise<Model> {
+    if (printGltf) console.warn("Loaded", gltfId, model);
+
+    // Handle loader was registered but no model was returned - should not completely break the engine
+    if (model == null) {
+        console.error(`Loaded model is null '${gltfId}' - please make sure the loader is registered correctly`);
+        return {
+            scene: new Object3D(),
+            animations: [],
+            scenes: []
+        };
+    }
+
+    // Handle OBJ or FBX loader results
+    if (model instanceof Object3D) {
+        model = {
+            scene: model,
+            animations: model.animations,
+            scenes: [model]
+        }
+    }
+    // Handle STL loader results
+    else if (model instanceof BufferGeometry) {
+        const mat = new MeshStandardMaterial({
+            color: new Color(0xdddddd)
+        });
+        const mesh = new Mesh(model, mat);
+        model = {
+            scene: mesh,
+            animations: [],
+            scenes: [mesh]
+        }
+    }
+
+
+    // Remove query parameters from gltfId
+    if (gltfId.includes("?")) {
+        gltfId = gltfId.split("?")[0];
+    }
+
+    // assign animations of loaded glTF to all scenes
+    for (const scene of model.scenes) {
+        if (scene && !scene.animations?.length) {
+            scene.animations = [...model.animations];
+        }
+    }
+
+    // E.g. fbx material cleanup
+    postprocessLoadedFile(loader, model);
+
+    // load components
+    if (isGLTFModel(model)) {
+        invokeLoadedImportPluginHooks(gltfId, model, context);
+        await getLoader().createBuiltinComponents(context, gltfId, model, seed, componentsExtension || undefined);
+    }
+
+    // Warmup the scene
+    await compileAsync(model.scene, context, context.mainCamera);
+
+    return model;
 }
+
+async function compileAsync(scene: Object3D, context: Context, camera?: Camera | null) {
+    if (!camera) camera = context.mainCamera;
+    try {
+        if (camera) {
+            await context.renderer.compileAsync(scene, camera, context.scene)
+                .catch(err => {
+                    console.warn(err.message);
+                });
+        }
+        else
+            registerPrewarmObject(scene, context);
+    }
+    catch (err: Error | any) {
+        console.warn(err?.message || err);
+    }
+}
+
+function checkIfUserAttemptedToLoadALocalFile(url: string) {
+    const fullurl = new URL(url, window.location.href).href;
+    if (fullurl.startsWith("file://")) {
+        const msg = "Hi - it looks like you are trying to load a local file which will not work. You need to use a webserver to serve your files.\nPlease refer to the documentation on <a href=\"https://fwd.needle.tools/needle-engine/docs/local-server\">https://docs.needle.tools</a> or ask for help in our <a href=\"https://discord.needle.tools\">discord community</a>";
+        showBalloonMessage(msg);
+        console.warn(msg)
+    }
+}
+
+// function _downloadGltf(data: string | ArrayBuffer) {
+//     if (typeof data === "string") {
+//         const a = document.createElement("a") as HTMLAnchorElement;
+//         a.href = data;
+//         a.download = data.split("/").pop()!;
+//         a.click();
+//     }
+//     else {
+//         const blob = new Blob([data], { type: "application/octet-stream" });
+//         const url = window.URL.createObjectURL(blob);
+//         const a = document.createElement("a") as HTMLAnchorElement;
+//         a.href = url;
+//         a.download = "download.glb";
+//         a.click();
+//     }
+// }
+
+/**
+ * Postprocess the loaded file. This is used to apply any custom postprocessing to the loaded file.
+ */
+function postprocessLoadedFile(loader: object, result: Model) {
+
+    if (loader instanceof FBXLoader || loader instanceof OBJLoader) {
+
+        let obj: Object3D | Model = result;
+        if (!(obj instanceof Object3D)) {
+            obj = (result as GLTF).scene;
+        }
+
+        obj.traverse((child) => {
+            const mesh = child as Mesh;
+            // See https://github.com/needle-tools/three.js/blob/b8df3843ff123ac9dc0ed0d3ccc5b568f840c804/examples/webgl_loader_multiple.html#L377
+            if (mesh?.isMesh) {
+                postprocessFBXMaterials(mesh, mesh.material as Material);
+            }
+        });
+    }
+}
+