@needle-tools/engine 2.59.2-pre → 2.60.0-pre

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@needle-tools/engine",
-    "version": "2.59.2-pre",
+    "version": "2.60.0-pre",
     "description": "Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development, and can be deployed anywhere. It is flexible, extensible, and collaboration and XR come naturally.",
     "main": "dist/needle-engine.umd.cjs",
     "module": "dist/needle-engine.js",
@@ -56,7 +56,7 @@
     "devDependencies": {
         "@babel/runtime": "^7.16.0",
         "@luncheon/esbuild-plugin-gzip": "^0.1.0",
-        "@needle-tools/gltf-transform-extensions": "^0.10.9-pre",
+        "@needle-tools/gltf-transform-extensions": "^0.10.10-pre",
         "@needle-tools/needle-component-compiler": "1.9.3",
         "@needle-tools/helper": "^0.2.1-pre",
         "@types/three": "0.146.0",

package/src/engine/engine_gltf_builtin_components.ts

@@ -9,7 +9,7 @@ import { assign, ImplementationInformation, ISerializable, SerializationContext
 import { NEEDLE_components } from "./extensions/NEEDLE_components";
 import { debugExtension } from "./engine_default_parameters";
 import { builtinComponentKeyName } from "./engine_constants";
-import { SourceIdentifier } from "./engine_types";
+import { GuidsMap, SourceIdentifier } from "./engine_types";
 import { UIDProvider } from "./engine_types";
 import { addNewComponent } from "./engine_components";
 import { getParam } from "./engine_utils";
@@ -83,9 +83,10 @@ export async function createBuiltinComponents(context: Context, gltfId: SourceId
         // and this should run before awake and onenable of newly created components
         if (idProvider) {
             // TODO: should we do this after setup callbacks now?
-            recursiveCreateGuids(gltf, idProvider);
+            const guidsMap: GuidsMap = {};
+            recursiveCreateGuids(gltf, idProvider, guidsMap);
             for (const scene of gltf.scenes)
-                recursiveCreateGuids(scene, idProvider);
+                recursiveCreateGuids(scene, idProvider, guidsMap);
         }
     });
 
@@ -94,22 +95,29 @@ export async function createBuiltinComponents(context: Context, gltfId: SourceId
     // console.log("finished creating builtin components", gltf.scene?.name, gltf);
 }
 
-function recursiveCreateGuids(obj: GameObject, idProvider: UIDProvider | null) {
+function recursiveCreateGuids(obj: GameObject, idProvider: UIDProvider | null, guidsMap: GuidsMap) {
     if (idProvider === null) return;
     if (!obj) return;
+    const prev = obj.guid;
     obj.guid = idProvider.generateUUID();
+    if (prev && prev !== "invalid")
+        guidsMap[prev] = obj.guid;
     // console.log(obj);
     if (obj && obj.userData && obj.userData.components) {
         for (const comp of obj.userData.components) {
             if (comp === null) continue;
+            const prev = comp.guid;
             comp.guid = idProvider.generateUUID();
-            // console.log(comp.guid)
+            if (prev && prev !== "invalid")
+                guidsMap[prev] = comp.guid;
+            if (comp.resolveGuids)
+                comp.resolveGuids(guidsMap);
         }
 
     }
     if (obj.children) {
         for (const child of obj.children) {
-            recursiveCreateGuids(child as GameObject, idProvider);
+            recursiveCreateGuids(child as GameObject, idProvider, guidsMap);
         }
     }
 }

package/src/engine/engine_serialization_builtin_serializer.ts

@@ -70,7 +70,23 @@ class ObjectSerializer extends TypeSerializer {
         return undefined;
     }
 
-    onDeserialize(data: ObjectData, context: SerializationContext) {
+    onDeserialize(data: ObjectData | string | null, context: SerializationContext) {
+
+        if (typeof data === "string") {
+            // ACTUALLY: this is already handled by the extension_utils where we resolve json pointers recursively
+            // if(data.startsWith("/nodes/")){
+            //     const node = parseInt(data.substring("/nodes/".length));
+            //     if (context.nodeToObject) {
+            //         const res = context.nodeToObject[node];
+            //         if (debugExtension)
+            //             console.log("Deserialized object reference?", data, res, context?.nodeToObject);
+            //         if (!res) console.warn("Did not find node: " + data, context.nodeToObject, context.object);
+            //         return res;
+            //     }
+            // }
+            return undefined;
+        }
+
         if (data) {
             if (data.node !== undefined && context.nodeToObject) {
                 const res = context.nodeToObject[data.node];

package/src/engine/engine_types.ts

@@ -151,6 +151,7 @@ export declare interface ICamera extends IComponent {
     nearClipPlane: number;
     farClipPlane: number;
     backgroundColor: RGBAColor | null;
+    backgroundBlurriness: number | undefined;
     clearFlags: number;
     aspect: number;
     fieldOfView: number;

package/src/engine/extensions/NEEDLE_render_objects.ts

@@ -45,6 +45,8 @@ declare type StencilSettingsModel = {
     zFailOp: number;
 }
 
+const $stencils = Symbol("stencils");
+
 export class NEEDLE_render_objects implements GLTFLoaderPlugin {
 
     private static stencils: { [key: string]: StencilSettingsModel[] } = {};
@@ -52,31 +54,33 @@ export class NEEDLE_render_objects implements GLTFLoaderPlugin {
     static applyStencil(obj?: IRenderer | null) {
         if (!obj) return;
         const source = obj.sourceId;
-        if (debug)
-            console.log(source, NEEDLE_render_objects.stencils);
+        if (debug) console.log(source, NEEDLE_render_objects.stencils);
         if (!source) return;
         const settings = NEEDLE_render_objects.stencils[source];
         if (!settings) return;
         for (let i = settings.length - 1; i >= 0; i--) {
             const stencil: StencilSettingsModel = settings[i];
             if (matchesLayer(stencil.layer, obj)) {
-                if (debug)
-                    console.log(stencil);
-                const mat = obj.sharedMaterial?.clone();
-                if (mat) {
-                    obj.sharedMaterial = mat;
-                    // you can have 50 renderer features per event until this breaks
-                    obj.gameObject.renderOrder = stencil.event * 1000 + stencil.index * 50;
-
-                    mat.stencilWrite = true;
-                    mat.stencilWriteMask = 255;
-                    mat.stencilFuncMask = 255;
-                    mat.stencilRef = stencil.value;
-                    mat.stencilFunc = stencil.compareFunc;
-                    mat.stencilZPass = stencil.passOp;
-                    mat.stencilFail = stencil.failOp;
-                    mat.stencilZFail = stencil.zFailOp;
+                if (debug) console.log(stencil);
+                for (let i = 0; i < obj.sharedMaterials.length; i++) {
+                    let mat = obj.sharedMaterials[i];
+                    if (mat) {
+                        // if (!mat[$stencils]) 
+                        mat = mat.clone();
+                        mat[$stencils] = true;
+                        mat.stencilWrite = true;
+                        mat.stencilWriteMask = 255;
+                        mat.stencilFuncMask = 255;
+                        mat.stencilRef = stencil.value;
+                        mat.stencilFunc = stencil.compareFunc;
+                        mat.stencilZPass = stencil.passOp;
+                        mat.stencilFail = stencil.failOp;
+                        mat.stencilZFail = stencil.zFailOp;
+                        obj.sharedMaterials[i] = mat;
+                    }
                 }
+                // you can have 50 renderer features per event until this breaks
+                obj.gameObject.renderOrder = stencil.event * 1000 + stencil.index * 50;
                 break;
             }
         }

package/src/engine-components/ParticleSystem.ts

@@ -73,12 +73,22 @@ export class ParticleSystemRenderer extends Behaviour {
         return this.particleMaterial;
     }
 
-    getMesh() {
+    getMesh(renderMode?: ParticleSystemRenderMode) {
         let geo: BufferGeometry | null = null;
-        if (this.particleMesh instanceof Mesh) {
-            geo = this.particleMesh.geometry;
+        if (renderMode === ParticleSystemRenderMode.HorizontalBillboard) {
+            geo = new THREE.BoxGeometry(1, 1, 0);
         }
-        if (geo === null) geo = new THREE.BoxGeometry(1, 1, 1);
+        else if (renderMode === ParticleSystemRenderMode.VerticalBillboard) {
+            geo = new THREE.BoxGeometry(1, 0, 1);
+        }
+
+        if (!geo) {
+            if (this.particleMesh instanceof Mesh) {
+                geo = this.particleMesh.geometry;
+            }
+            if (geo === null) geo = new THREE.BoxGeometry(1, 1, 0);
+        }
+
         const res = new Mesh(geo, this.getMaterial());
         return res;
     }
@@ -494,7 +504,7 @@ class ParticleSystemInterface implements ParticleSystemParameters {
     onlyUsedByOther?: boolean | undefined;
     readonly behaviors: Behavior[] = [];
     get instancingGeometry() {
-        return this.system.renderer.getMesh().geometry;
+        return this.system.renderer.getMesh(this.system.renderer.renderMode).geometry;
     }
     get renderMode() {
         if (this.system.trails["enabled"] === true) {
@@ -503,8 +513,8 @@ class ParticleSystemInterface implements ParticleSystemParameters {
         switch (this.system.renderer.renderMode) {
             case ParticleSystemRenderMode.Billboard: return RenderMode.BillBoard;
             // case ParticleSystemRenderMode.Stretch: return RenderMode.Stretch;
-            // case ParticleSystemRenderMode.HorizontalBillboard: return RenderMode.HorizontalBillboard;
-            // case ParticleSystemRenderMode.VerticalBillboard: return RenderMode.VerticalBillboard;
+            case ParticleSystemRenderMode.HorizontalBillboard: return RenderMode.LocalSpace;
+            case ParticleSystemRenderMode.VerticalBillboard: return RenderMode.LocalSpace;
             case ParticleSystemRenderMode.Mesh: return RenderMode.LocalSpace;
         }
         return RenderMode.BillBoard;

package/src/engine-components/ParticleSystemModules.ts

@@ -36,8 +36,8 @@ export interface IParticleSystem {
 export enum ParticleSystemRenderMode {
     Billboard = 0,
     //   Stretch = 1,
-    //   HorizontalBillboard = 2,
-    //   VerticalBillboard = 3,
+    HorizontalBillboard = 2,
+    VerticalBillboard = 3,
     Mesh = 4,
     //   None = 5,
 }
@@ -1117,7 +1117,7 @@ export class RotationOverLifetimeModule {
     @serializable()
     zMultiplier!: number;
 
-    evaluate(t01: number, t:number): number {
+    evaluate(t01: number, t: number): number {
         if (!this.enabled) return 0;
         if (!this.separateAxes) {
             const rot = this.z.evaluate(t01, t) * -1;

package/src/engine-components/Skybox.ts

@@ -18,9 +18,6 @@ export class RemoteSkybox extends Behaviour {
     @serializable()
     background: boolean = true;
 
-    @serializable()
-    backgroundBlurriness: number = 0;
-
     @serializable()
     environment: boolean = true;
 
@@ -84,7 +81,8 @@ export class RemoteSkybox extends Behaviour {
         this._prevLoadedEnvironment = envMap;
         const nameIndex = url.lastIndexOf("/");
         envMap.name = url.substring(nameIndex >= 0 ? nameIndex + 1 : 0);
-        this.context.scene.backgroundBlurriness = this.backgroundBlurriness;
+        if (this.context.mainCameraComponent?.backgroundBlurriness !== undefined)
+            this.context.scene.backgroundBlurriness = this.context.mainCameraComponent.backgroundBlurriness;
     }
 
 

package/src/engine-components/timeline/PlayableDirector.ts

@@ -8,6 +8,7 @@ import * as Models from "./TimelineModels";
 import * as Tracks from "./TimelineTracks";
 import { deepClone, getParam } from '../../engine/engine_utils';
 import { GuidsMap } from '../../engine/engine_types';
+import { Object3D } from 'three';
 
 const debug = getParam("debugtimeline");
 
@@ -371,10 +372,17 @@ export class PlayableDirector extends Behaviour {
             }
             // only handle animation tracks
             if (track.type === Models.TrackType.Animation) {
-                if (track.clips.length <= 0) continue;
+                if (track.clips.length <= 0) {
+                    if(debug) console.warn("Animation track has no clips", track);
+                    continue;
+                }
                 // loop outputs / bindings, they should contain animator references
                 for (let i = track.outputs.length - 1; i >= 0; i--) {
-                    const binding = track.outputs[i] as Animator;
+                    let binding = track.outputs[i] as Animator;
+                    if(binding instanceof Object3D){
+                        const anim = GameObject.getOrAddComponent(binding, Animator);
+                        if(anim) binding = anim;
+                    }
                     if (typeof binding.enabled === "boolean") binding.enabled = false;
                     const animationClips = binding?.gameObject?.animations;
                     if (animationClips) {

package/src/engine-components/timeline/TimelineModels.ts

@@ -51,7 +51,8 @@ export declare type ClipModel = {
     easeInDuration: number;
     easeOutDuration: number;
     preExtrapolationMode: ClipExtrapolation;
-    postExtrapolationMode: ClipExtrapolation
+    postExtrapolationMode: ClipExtrapolation;
+    reversed?: boolean;
 }
 
 export declare type AnimationClipModel = {
@@ -59,8 +60,8 @@ export declare type AnimationClipModel = {
     loop: boolean;
     duration: number;
     removeStartOffset: boolean;
-    position: Vec3 | THREE.Vector3;
-    rotation: Quat | THREE.Quaternion;
+    position?: Vec3 | THREE.Vector3;
+    rotation?: Quat | THREE.Quaternion;
 }
 
 export declare type AudioClipModel = {

package/src/engine-components/timeline/TimelineTracks.ts

@@ -213,7 +213,7 @@ export class AnimationTrackHandler extends TrackHandler {
             const clipData = model.asset as Models.AnimationClipModel;
             const pos = clipData.position as any;
             const rot = clipData.rotation as any;
-            if (pos.x !== undefined) {
+            if (pos && pos.x !== undefined) {
                 if (!pos.isVector3) {
                     clipData.position = new Vector3(pos.x, pos.y, pos.z);
                 }
@@ -254,6 +254,7 @@ export class AnimationTrackHandler extends TrackHandler {
     private _tempPos = new THREE.Vector3();
     private _summedRot = new THREE.Quaternion();
     private _tempRot = new THREE.Quaternion();
+    private _clipRotQuat = new THREE.Quaternion();
 
     evaluate(time: number) {
         if (this.track.muted) return;
@@ -315,7 +316,9 @@ export class AnimationTrackHandler extends TrackHandler {
                     }
                 }
 
-                action.time = t;
+                if(model.reversed === true) action.time = action.getClip().duration - t;
+                else action.time = t;
+                
                 action.timeScale = 0;
                 const effectiveWeight = weight * this.director.weight;
                 action.weight = effectiveWeight;
@@ -335,8 +338,10 @@ export class AnimationTrackHandler extends TrackHandler {
                     const tempRot = this._tempRot.identity();
 
                     const clipOffsetRot = clipModel.rotation as Quaternion;
-                    const clipRot = new THREE.Quaternion();
-                    clipRot.slerp(clipOffsetRot, weight);
+                    if (clipOffsetRot) {
+                        this._clipRotQuat.identity();
+                        this._clipRotQuat.slerp(clipOffsetRot, weight);
+                    }
 
                     const offsets = this._actionOffsets[i];
                     if (offsets.hasOffsets) {
@@ -345,7 +350,10 @@ export class AnimationTrackHandler extends TrackHandler {
                                 tempPos.copy(offsets.rootPositionOffset);
                             else tempPos.set(0, 0, 0);
 
-                            tempPos.applyQuaternion(summedRot).applyQuaternion(clipRot)
+                            tempPos.applyQuaternion(summedRot);
+                            if (this._clipRotQuat)
+                                tempPos.applyQuaternion(this._clipRotQuat);
+
                             if (offsets.rootQuaternionOffset) {
                                 // console.log(new THREE.Euler().setFromQuaternion(offsets.rootQuaternionOffset).y.toFixed(2));
                                 tempRot.copy(offsets.rootQuaternionOffset);
@@ -355,10 +363,12 @@ export class AnimationTrackHandler extends TrackHandler {
                         }
                     }
 
-                    totalRotation.multiply(clipRot);
+                    if (this._clipRotQuat)
+                        totalRotation.multiply(this._clipRotQuat);
                     totalRotation.multiply(summedRot);
 
-                    summedPos.add(clipModel.position as THREE.Vector3);
+                    if (clipModel.position)
+                        summedPos.add(clipModel.position as THREE.Vector3);
                     totalPosition.add(summedPos);
                 }
 
@@ -506,7 +516,7 @@ export class AudioTrackHandler extends TrackHandler {
         }
     }
     evaluate(time: number) {
-        if(muteAudioTracks) return;
+        if (muteAudioTracks) return;
         if (this.track.muted) return;
         for (let i = 0; i < this.models.length; i++) {
             const model = this.models[i];
@@ -521,6 +531,19 @@ export class AudioTrackHandler extends TrackHandler {
                     audio.offset = model.clipIn + (time - model.start) * model.timeScale;
                     audio.play();
                 }
+                else {
+                    const targetOffset = model.clipIn + (time - model.start) * model.timeScale;
+                    // seems it's non-trivial to get the right time from audio sources;
+                    // https://github.com/mrdoob/three.js/blob/master/src/audio/Audio.js#L170
+                    const currentTime = audio.context.currentTime - audio["_startedAt"] + audio.offset;
+                    const diff = Math.abs(targetOffset - currentTime);
+
+                    if (diff > 0.3) {
+                        audio.offset = targetOffset;
+                        audio.stop();
+                        audio.play();
+                    }
+                }
                 let vol = model.asset.volume;
                 if (model.easeInDuration > 0) {
                     const easeIn = Math.min((time - model.start) / model.easeInDuration, 1);