@needle-tools/engine 2.29.0-pre → 2.31.0-pre

package/src/engine/engine_setup.ts

@@ -72,6 +72,11 @@ export enum FrameEvent {
 	PhysicsStep = 10,
 }
 
+export enum XRSessionMode {
+	ImmersiveVR = "immersive-vr",
+	ImmersiveAR = "immersive-ar",
+}
+
 export declare type OnBeforeRenderCallback = (renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group) => void
 
 export class Context {
@@ -109,6 +114,7 @@ export class Context {
 	get domX(): number { return this.domElement.offsetLeft; }
 	get domY(): number { return this.domElement.offsetTop; }
 	get isInXR() { return this.renderer.xr?.isPresenting || false; }
+	xrSessionMode : XRSessionMode | undefined = undefined;
 	get xrSession() { return this.renderer.xr?.getSession(); }
 	get arOverlayElement(): HTMLElement {
 		const el = this.domElement as any;

package/src/engine/engine_utils.ts

@@ -181,17 +181,43 @@ export function delay(milliseconds: number): Promise<void> {
 // if a timeline is exported via menu item the audio clip path is relative to the glb (same folder)
 // we need to detect that here and build the new audio source path relative to the new glb location
 // the same is/might be true for any file that is/will be exported via menu item
-export function getPath(glbLocation: SourceIdentifier | undefined, path: string) {
-    if (path && glbLocation && !path.includes("/")) {
-        // get directory of glb and prepend it to the audio file path
-        const pathIndex = glbLocation.lastIndexOf("/");
-        if (pathIndex >= 0) {
-            const newPath = glbLocation.substring(0, pathIndex + 1) + path;
-            return newPath;
+const debugGetPath = getParam("debugsourcepath");
+export function getPath(source: SourceIdentifier | undefined, uri: string): string {
+    if (source === undefined) {
+        if (debugGetPath) console.warn("getPath: source is undefined, returning uri", uri);
+        return uri;
+    }
+    if (uri.startsWith("http")) {
+        if (debugGetPath) console.warn("getPath: uri is absolute, returning uri", uri);
+        return uri;
+    }
+    const pathIndex = source.lastIndexOf("/");
+    if (pathIndex >= 0) {
+        let newUri = source.substring(0, pathIndex + 1);
+
+        const uriDirectoryIndex = uri.lastIndexOf("/");
+        if (uriDirectoryIndex >= 0) {
+            newUri += uri.substring(uriDirectoryIndex + 1);
+        } else {
+            newUri += uri;
         }
+        if (debugGetPath) console.log("getPath:", source, " - changed uri from\n", uri, "\n→ ", newUri);
+        return newUri;
     }
-    return path;
+    return uri;
 }
+// export function getPath(glbLocation: SourceIdentifier | undefined, path: string) {
+//     if (path && glbLocation && !path.includes("/")) {
+//         // get directory of glb and prepend it to the audio file path
+//         const pathIndex = glbLocation.lastIndexOf("/");
+//         if (pathIndex >= 0) {
+//             const newPath = glbLocation.substring(0, pathIndex + 1) + path;
+//             return newPath;
+//         }
+//     }
+//     return path;
+// }
+
 
 export type WriteCallback = (data: any) => void;
 

package/src/engine/extensions/NEEDLE_deferred_texture.ts

@@ -3,7 +3,7 @@ import { GLTF, GLTFLoader, GLTFLoaderPlugin, GLTFParser } from "three/examples/j
 import { SourceIdentifier } from "../engine_gltf";
 import { Context } from "../engine_setup";
 import { addDracoAndKTX2Loaders } from "../engine_loaders";
-import { getParam } from "../engine_utils";
+import { getParam, getPath } from "../engine_utils";
 
 export const EXTENSION_NAME = "NEEDLE_deferred_texture";
 
@@ -11,24 +11,27 @@ const debug = getParam("debugdeferredtextures")
 
 declare type DeferredTextureModel = {
     uri: string;
-    guid?: string;
+    guid: string;
     usage?: string,
 }
 
 export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
 
-    static assignTextureLOD(context: Context, material: Material, level: number = 0) {
+    static assignTextureLOD(context: Context, source : SourceIdentifier | undefined, material: Material, level: number = 0) {
         if (!material) return;
         for (const slot of Object.keys(material)) {
             const val = material[slot];
             if (val?.isTexture === true) {
                 if (debug)
-                    console.log("FIND", slot, val?.name, val?.userData, val);
-                NEEDLE_deferred_texture.getOrLoadTexture(context, material, slot, val, level).then(t => {
+                    console.log("-----------\n", "FIND", material.name, slot, val?.name, val?.userData, val);
+                NEEDLE_deferred_texture.getOrLoadTexture(context, source, material, slot, val, level).then(t => {
                     if (t?.isTexture === true) {
                         t.needsUpdate = true;
+                        if (debug)
+                            console.log("Assign LOD", material.name, slot, t.name, t["guid"], material, "Prev:", val, "Now:", t, "\n--------------");
                         material[slot] = t;
                         material.needsUpdate = true;
+
                     }
                 });
             }
@@ -67,9 +70,8 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
     // }
 
     afterRoot(gltf: GLTF): null {
-
         if (debug)
-            console.log("AFTER", gltf);
+            console.log("AFTER", this.sourceId, gltf);
         this.parser.json.textures?.forEach((textureInfo, index) => {
             if (textureInfo?.extensions) {
                 const ext: DeferredTextureModel = textureInfo?.extensions[EXTENSION_NAME];
@@ -91,29 +93,31 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
     private static cache = new Map<string, DeferredTextureModel>();
     private static resolved: { [key: string]: Texture } = {};
 
-    private static async getOrLoadTexture(context: Context, material: Material, slot: string, current: Texture, _level: number): Promise<Texture | null> {
+    private static async getOrLoadTexture(context: Context, source : SourceIdentifier | undefined, material: Material, slot: string, current: Texture, _level: number): Promise<Texture | null> {
 
         const key = current.uuid;
-        const ext: DeferredTextureModel = NEEDLE_deferred_texture.cache.get(key) || current.userData.deferred;
+        const ext: DeferredTextureModel | undefined = NEEDLE_deferred_texture.cache.get(key);// || current.userData.deferred;
         if (ext) {
             if (debug)
                 console.log(key, ext.uri, ext.guid);
-            const uri = ext.uri;
+            const uri = getPath(source, ext.uri);
             if (uri.endsWith(".glb") || uri.endsWith(".gltf")) {
                 if (!ext.guid) {
                     console.warn("missing pointer for glb/gltf texture", ext);
                     return null;
                 }
-                const resolveKey = ext.uri + "_" + ext.guid;
-                if (this.resolved[resolveKey])
+                const resolveKey = uri + "_" + ext.guid;
+                if (this.resolved[resolveKey]) {
+                    if (debug) console.log("Texture has already been loaded: " + resolveKey, material.name, slot, current.name);
                     return this.resolved[resolveKey];
+                }
 
                 const loader = new GLTFLoader();
                 addDracoAndKTX2Loaders(loader, context);
-                if (debug) console.log("Load " + uri, material.name, slot);
+                if (debug) console.log("Load " + uri, material.name, slot, ext.guid);
                 const gltf = await loader.loadAsync(uri);
                 const parser = gltf.parser;
-
+                if (debug) console.log("Loading finished " + uri, material.name, slot, ext.guid);
                 let index = -1;
                 let found = false;
                 for (const tex of gltf.parser.json.textures) {
@@ -137,20 +141,22 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
 
                 // const index = Number.parseInt(ext.pointer.substring("textures/".length));
                 const tex = await parser.getDependency("texture", index);
+                if (tex) {
+                    tex.guid = ext.guid;
+                }
                 this.resolved[resolveKey] = tex as Texture;
-
                 if (debug)
-                    console.log(material.name, slot, "change \"" + current.name + "\" → \"" + tex.name + "\"", uri, index, tex, material);
+                    console.log(material.name, slot, "change \"" + current.name + "\" → \"" + tex.name + "\"", uri, index, tex, material, resolveKey);
                 return tex;
             }
             else {
-
+                if (debug) console.log("Load texture from uri: " + uri);
                 const loader = new TextureLoader();
-                const loaded = await loader.loadAsync(ext.uri);
+                const loaded = await loader.loadAsync(uri);
                 if (loaded && debug) {
                     console.log(ext, loaded);
                 }
-                else console.warn("failed loading", ext.uri);
+                else console.warn("failed loading", uri);
                 return loaded;
             }
             // loader.then((h: Texture) => {

package/src/engine/extensions/extension_utils.ts

@@ -10,13 +10,20 @@ declare type DependencyInfo = {
     dependencyName: string,
 }
 
-const rootExtensionPrefix = "/extensions/";
+const rootExtensionPrefix = ["/extensions/", "extensions/"];
 const defaultDependencies = [
     { prefix: "/nodes/", dependencyName: "node" },
     { prefix: "/meshes/", dependencyName: "mesh" },
     { prefix: "/materials/", dependencyName: "material" },
     { prefix: "/textures/", dependencyName: "texture" },
-    { prefix: "/animations/", dependencyName: "animation" }
+    { prefix: "/animations/", dependencyName: "animation" },
+
+    // legacy support
+    { prefix: "nodes/", dependencyName: "node" },
+    { prefix: "meshes/", dependencyName: "mesh" },
+    { prefix: "materials/", dependencyName: "material" },
+    { prefix: "textures/", dependencyName: "texture" },
+    { prefix: "animations/", dependencyName: "animation" },
 ]
 
 export async function resolveReferences(parser: GLTFParser, obj) {
@@ -95,16 +102,21 @@ function internalResolve(paths: DependencyInfo[], parser: GLTFParser, obj, promi
 
 
 function resolveExtension(parser: GLTFParser, str): Promise<void> | null {
-    if (parser && parser.plugins && typeof str === "string" && str.startsWith(rootExtensionPrefix)) {
-        let name = str.substring(rootExtensionPrefix.length);
-        const endIndex = name.indexOf("/");
-        if (endIndex >= 0) name = name.substring(0, endIndex);
-        const ext = parser.plugins[name] as IExtensionReferenceResolver;
-        if (debugExtension)
-            console.log(name, ext);
-        if (typeof ext?.resolve === "function") {
-            const path = str.substring(rootExtensionPrefix.length + name.length + 1);
-            return ext.resolve(parser, path);
+    if (parser && parser.plugins && typeof str === "string") {
+        for (const prefix of rootExtensionPrefix) {
+            if (str.startsWith(prefix)) {
+                let name = str.substring(prefix.length);
+                const endIndex = name.indexOf("/");
+                if (endIndex >= 0) name = name.substring(0, endIndex);
+                const ext = parser.plugins[name] as IExtensionReferenceResolver;
+                if (debugExtension)
+                    console.log(name, ext);
+                if (typeof ext?.resolve === "function") {
+                    const path = str.substring(prefix.length + name.length + 1);
+                    return ext.resolve(parser, path);
+                }
+                break;
+            }
         }
     }
     return null;

package/src/engine/extensions/extensions.ts

@@ -26,8 +26,9 @@ export function registerComponentExtension(loader: GLTFLoader): NEEDLE_component
 class PointerResolver {
     resolvePath(path: string) {
         if (path.includes('/extensions/builtin_components/'))
-            path = path.replace('/extensions/builtin_components/', '/userData/components/');
-
+            return path.replace('/extensions/builtin_components/', '/userData/components/');
+        if (path.includes('extensions/builtin_components/'))
+            return path.replace('extensions/builtin_components/', '/userData/components/');
         return path;
     }
 }

package/src/engine-components/Camera.ts

@@ -5,6 +5,7 @@ import { getParam } from "../engine/engine_utils";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { RGBAColor } from "./js-extensions/RGBAColor";
 import { PerspectiveCamera } from "three";
+import { XRSessionMode } from "../engine/engine_setup";
 
 export enum ClearFlags {
     Skybox = 1,
@@ -159,7 +160,6 @@ export class Camera extends Behaviour {
     }
 
     applyClearFlagsIfIsActiveCamera() {
-        // TODO: need to differentiate between what kind of XR (set skybox in VR but not in mobile AR for example!?)
         if (this._cam && this.context.mainCameraComponent === this) {
             switch (this._clearFlags) {
                 case ClearFlags.Skybox:
@@ -192,13 +192,20 @@ export class Camera extends Behaviour {
     }
 
     private environmentIsTransparent(): boolean {
-        const session = this.context.renderer.xr?.getSession()
+        const session = this.context.renderer.xr?.getSession();
         if (!session) return false;
         const environmentBlendMode = session.environmentBlendMode;
         const transparent = environmentBlendMode === 'additive' || environmentBlendMode === 'alpha-blend';
+        // workaround for Quest 2 returning opaque when it should be alpha-blend
+        // check user agent if this is the Quest browser and return true if so
+
+        if (environmentBlendMode === "opaque" && navigator.userAgent?.includes("OculusBrowser")) {
+            if (this.context.xrSessionMode === XRSessionMode.ImmersiveAR) return true;
+        }
         return transparent;
     }
 
+
     private enableSkybox() {
         if (!this._skybox)
             this._skybox = new CameraSkybox(this);

package/src/engine-components/Component.ts

@@ -9,6 +9,7 @@ import * as main from "../engine/engine_mainloop_utils";
 import { Object3D } from "three";
 import { InstantiateIdProvider, syncDestroy, syncInstantiate } from "../engine/engine_networking_instantiate";
 import { SourceIdentifier } from "../engine/engine_gltf";
+import { Collision } from "../engine/engine_physics";
 
 export interface UIDProvider {
     seed: number;
@@ -835,6 +836,10 @@ class Component implements EventTarget {
     onBeforeRender?(frame: THREE.XRFrame | null): void;
     onAfterRender?(): void;
 
+    onCollisionEnter?(col: Collision);
+    onCollisionExit?(col: Collision);
+    onCollisionStay?(col: Collision);
+
     startCoroutine(routine: Generator, evt: FrameEvent = FrameEvent.Update): Generator {
         return this.context.registerCoroutineUpdate(this, routine, evt);
     }
@@ -865,6 +870,7 @@ class Component implements EventTarget {
         this.__didEnable = false;
         this.__isEnabled = undefined;
         this.__destroyed = false;
+        this.__internalResetsCachedPhysicsData();
     }
 
     __internalAwake() {
@@ -892,6 +898,7 @@ class Component implements EventTarget {
     __internalDisable() {
         if (!this.__didEnable) return;
         this.__didEnable = false;
+        this._collisionExitRoutine = undefined;
         this.onDisable();
     }
 
@@ -899,6 +906,57 @@ class Component implements EventTarget {
         this.__internalNewInstanceCreated();
     }
 
+    private _collisionExitRoutine: any;
+    private _collisions: Map<Object3D, { col: Collision, frame: number }> | null = null;
+
+    private __internalResetsCachedPhysicsData() {
+        this._collisionExitRoutine = null;
+        this._collisions = null;
+    }
+
+    __internalHandleCollision(col: Collision) {
+        if (this.onCollisionEnter || this.onCollisionExit || this.onCollisionStay) {
+            const otherObject = col.gameObject;
+            if (!this._collisions) this._collisions = new Map();
+            if (this._collisions.has(otherObject)) {
+                const cur = this._collisions.get(otherObject)!;
+                cur.frame = this.context.time.frameCount;
+                cur.col = col;
+                this.onCollisionStay?.call(this, col);
+            }
+            else {
+                const entry = { col, frame: this.context.time.frameCount };
+                this._collisions.set(otherObject, entry);
+                this.onCollisionEnter?.call(this, col);
+            }
+        }
+    }
+
+    sendExitCollisionEvent(obj : Object3D) {
+        if(!this._collisions) return;
+        const col = this._collisions.get(obj);
+        if(!col) return;
+        this.onCollisionExit?.call(this, col.col);
+        this._collisions.delete(obj);
+    }
+
+    private __waitForCollisionExit() {
+        if (this._collisionExitRoutine) return;
+        // const self = this;
+        // function* routine() {
+        //     while (self._collisions && self._collisions.size > 0) {
+        //         for (let ob of self._collisions.keys()) {
+        //             const entry = self._collisions!.get(ob)!;
+        //             if (self.context.time.frameCount - entry.frame > 1) {
+        //                 self._collisions!.delete(ob);
+        //                 self.onCollisionExit?.call(self, entry.col);
+        //             }
+        //         }
+        //         yield;
+        //     }
+        // }
+        // this._collisionExitRoutine = this.startCoroutine(routine(), FrameEvent.EarlyUpdate);
+    }
 
 
     // TODO move this to threeutils

package/src/engine-components/OrbitControls.ts

@@ -102,7 +102,8 @@ export class OrbitControls extends Behaviour {
         if (this._controls) {
             const camGo = GameObject.getComponent(this.gameObject, Camera);
             if (camGo && !this.setFromTargetPosition()) {
-                console.log("NO TARGET");
+                if(this.debugLog)
+                    console.log("NO TARGET");
                 const forward = new THREE.Vector3(0, 0, -1).applyMatrix4(camGo.cam.matrixWorld);
                 this.setTarget(forward, true);
             }

package/src/engine-components/Renderer.ts

@@ -13,6 +13,7 @@ const suppressInstancing = getParam("noInstancing");
 const debugLightmap = getParam("debuglightmaps") ? true : false;
 const debugInstancing = getParam("debuginstancing");
 const debugProgressiveLoading = getParam("debugprogressiveload");
+const suppressProgressiveLoading = getParam("noprogressiveload");
 
 export class FieldWithDefault {
     public path: string | null = null;
@@ -172,6 +173,8 @@ export class Renderer extends Behaviour {
     }
 
     awake() {
+        this.clearInstancingState();
+
         const type = this.gameObject.type;
         if (type === "Group") {
             for (const child of this.gameObject.children) {
@@ -245,6 +248,11 @@ export class Renderer extends Behaviour {
     private handles: InstanceHandle[] | null | undefined = undefined;
     private prevLayers: number[] | null | undefined = undefined;
 
+    private clearInstancingState() {
+        this._isInstancingEnabled = false;
+        this.handles = undefined;
+        this.prevLayers = undefined;
+    }
     setInstancingEnabled(enabled: boolean): boolean {
         if (this._isInstancingEnabled === enabled) return enabled && (this.handles === undefined || this.handles != null && this.handles.length > 0);
         this._isInstancingEnabled = enabled;
@@ -338,16 +346,16 @@ export class Renderer extends Behaviour {
     onBeforeRenderThree(_renderer, _scene, _camera, _geometry, material, _group) {
 
         // progressive load before rendering so we only load textures for visible materials
-        if (material._didRequestTextureLOD === undefined) {
+        if (!suppressProgressiveLoading && material._didRequestTextureLOD === undefined) {
             material._didRequestTextureLOD = 0;
             if (debugProgressiveLoading) {
                 console.log("Load material LOD (with delay)", material.name);
                 setTimeout(() => {
-                    NEEDLE_deferred_texture.assignTextureLOD(this.context, material);
+                    NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
                 }, 2000);
             }
             else {
-                NEEDLE_deferred_texture.assignTextureLOD(this.context, material);
+                NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
             }
         }
 

package/src/engine-components/RigidBody.ts

@@ -29,7 +29,7 @@ export class Rigidbody extends Behaviour {
     @serializeable()
     angularDrag: number = 1;
     @serializeable()
-    detectCollision: boolean = true;
+    detectCollisions: boolean = true;
     @serializeable()
     sleepThreshold: number = 0.01;
 
@@ -106,7 +106,7 @@ export class Rigidbody extends Behaviour {
         options.angularDrag = this.angularDrag;
         options.mass = this.mass;
         options.kinematic = this.isKinematic;
-        options.physicsEvents = this.detectCollision;
+        options.physicsEvents = this.detectCollisions;
         options.sleepThreshold = this.sleepThreshold;
         this._body = this.context.physics.createBody(this.gameObject, options);
     }