@needle-tools/engine 4.10.0-next.55c0bf9 → 4.10.0-next.f5ce0ae

package/src/engine/webcomponents/needle-engine.ts

@@ -38,14 +38,18 @@ const observedAttributes = [
     "loadstart",
     "progress",
     "loadfinished",
+
     "dracoDecoderPath",
     "dracoDecoderType",
     "ktx2DecoderPath",
+
     "tone-mapping",
     "tone-mapping-exposure",
     "background-blurriness",
     "background-color",
     "environment-intensity",
+
+    "focus-rect",
 ]
 
 // https://developers.google.com/web/fundamentals/web-components/customelements
@@ -95,7 +99,7 @@ export class NeedleEngineWebComponent extends HTMLElement implements INeedleEngi
         if (value === null) this.removeAttribute("camera-controls");
         else this.setAttribute("camera-controls", value ? "true" : "false");
     }
-    
+
 
     /**
      * Get the current context for this web component instance. The context is created when the src attribute is set and the loading has finished.  
@@ -331,17 +335,17 @@ export class NeedleEngineWebComponent extends HTMLElement implements INeedleEngi
                 if (debug) console.log("ktx2DecoderPath", newValue);
                 setKtx2TranscoderPath(newValue);
                 break;
-                
+
             case "tonemapping":
             case "tone-mapping":
             case "tone-mapping-exposure":
             case "background-blurriness":
             case "background-color":
             case "environment-intensity":
-            {
-                this.applyAttributes();
-                break;
-            }
+                {
+                    this.applyAttributes();
+                    break;
+                }
             case "public-key": {
                 if (newValue != PUBLIC_KEY)
                     this.setPublicKey();
@@ -352,6 +356,25 @@ export class NeedleEngineWebComponent extends HTMLElement implements INeedleEngi
                     this.setVersion();
                 break;
             }
+
+            case "focus-rect":
+                {
+                    const focus_rect = this.getAttribute("focus-rect") as HTMLElement | string | null;
+                    if (focus_rect && this._context) {
+                        if (focus_rect === null) {
+                            this._context.setCameraFocusRect(null);
+                        }
+                        else if (typeof focus_rect === "string" && focus_rect.length > 0) {
+                            const element = document.querySelector(focus_rect);
+                            this._context.setCameraFocusRect(element instanceof HTMLElement ? element : null);
+                        }
+                        else if (focus_rect instanceof HTMLElement) {
+                            this._context.setCameraFocusRect(focus_rect);
+                        }
+
+                    }
+                }
+                break;
         }
     }
 
@@ -570,6 +593,10 @@ export class NeedleEngineWebComponent extends HTMLElement implements INeedleEngi
                 this._context.renderer.setClearColor(rgbaColor, rgbaColor.alpha);
                 this.context.scene.background = null;
             }
+            // HACK: if we set background-color to a color and then back to null we want the background-image attribute to re-apply
+            else if (this.getAttribute("background-image")) {
+                this.setAttribute("background-image", this.getAttribute("background-image")!);
+            }
         }
     }
 

package/src/engine/xr/NeedleXRController.ts

@@ -23,10 +23,10 @@ const debugCustomGesture = getParam("debugcustomgesture");
 // let _didReceiveSelectStartEvent = false;
 
 // https://github.com/immersive-web/webxr-input-profiles/blob/4484a05e30bcd43fe86bb4e06b7a707861a26796/packages/registry/profiles/meta/meta-quest-touch-plus.json
-declare type ControllerAxes = "xr-standard-thumbstick";
-declare type StickName = "xr-standard-thumbstick";
+declare type ControllerAxes = "xr-standard-thumbstick" | "xr-standard-touchpad";
+declare type StickName = "xr-standard-thumbstick" | "xr-standard-touchpad";
 declare type Mapping = "xr-standard";
-declare type ComponentType = "button" | "thumbstick" | "squeeze";
+declare type ComponentType = "button" | "thumbstick" | "squeeze" | "touchpad";
 declare type GamepadKey = "button" | "xAxis" | "yAxis";
 
 declare type NeedleXRControllerButtonName = ButtonName | "primary-button" | "primary";
@@ -407,6 +407,16 @@ C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inp
             gamepadStr += "\n[axes " + gp.axes.length + "]: " + gp.axes.map(a => a.toPrecision(1)).join(",");
             debugStr += "\n" + gamepadStr;
         }
+        if (this._layout) {
+            debugStr += "\nLayout: ";
+            for (const component of Object.keys(this._layout.components || {})) {
+                const val = this.getStick(component as StickName);
+                const indices = this._layout.components[component]?.gamepadIndices;
+                const indicesAsString = indices ? Object.entries(indices).map(e => e[0][0].toUpperCase() + e[0].slice(1) + "=" + e[1]).join(",") : "";
+                debugStr += `\n  ${component}: ${this._layout.components[component]?.type} [${indicesAsString}] (${val.x.toPrecision(2)},${val.y.toPrecision(2)})`;
+            }
+        }
+        
         Gizmos.DrawLabel(debugLabelPosition, debugStr, .006);
     }
 
@@ -730,6 +740,7 @@ C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inp
         if (componentModel?.gamepadIndices) {
             switch (componentModel.type) {
                 case "thumbstick":
+                case "touchpad":
                     if (this.inputSource.gamepad) {
                         const xIndex = componentModel.gamepadIndices!.xAxis!;
                         const yIndex = componentModel.gamepadIndices!.yAxis!;
@@ -760,7 +771,11 @@ C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inp
         this._isMetaQuestTouchController = this.profiles.includes("meta-quest-touch-plus") || this.profiles.includes("oculus-touch-v3");
 
         // Proper profile starting with v69 and browser 35.1
-        this._isMxInk = this.profiles.includes("logitech-mx-ink")
+        this._isMxInk = this.profiles.includes("logitech-mx-ink");
+
+        // For debugging to see ALL available profiles
+        /** @ts-ignore */
+        // fetchProfilesList(DEFAULT_PROFILES_PATH).then(list => console.log("Available controller profiles", list));
 
         if (!this._layout) {
             // Ignore transient-pointer since we likely don't want to spawn a controller visual just for a temporary pointer.
@@ -780,6 +795,8 @@ C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inp
                     res.assetPath || ""
                 );
 
+                // const overrideProfile = await fetch(DEFAULT_PROFILES_PATH + "/htc-vive-focus-3/profile.json").then(r => r.json());
+
                 const profile = res.profile as InputDeviceProfile;
                 const layout = profile.layouts[this.inputSource.handedness];
                 this._layout = layout;
@@ -791,6 +808,21 @@ C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inp
                             this._layout.gamepad[component.gamepadIndices!.button!] = key as XRControllerButtonName;
                         }
                     }
+
+                    // If we have 4 axes and no thumbstick defined, we define thumbstick for axis 3+4
+                    // This is a workaround for HTC Vive Focus 3 controllers, which have the profile for Vive Focus Plus...
+                    // This workaround fixes it for HTC Vive Focus 3 but does not change anything for Vive Focus Plus controllers
+                    if (this.profiles.length >= 1 && this.profiles[0] === "htc-vive-focus-plus") {
+                        if (this.inputSource.gamepad && this.inputSource.gamepad.axes.length === 4 && !this._layout.components["xr-standard-thumbstick"]) {
+                            this._layout.components["xr-standard-thumbstick"] = {
+                                type: "thumbstick",
+                                gamepadIndices: {
+                                    xAxis: 2,
+                                    yAxis: 3,
+                                }
+                            }
+                        }
+                    }
                 }
                 // if (debug) console.log(this._layout, this.inputSource);
                 // debugger;

package/src/engine-components/CameraUtils.ts

@@ -122,7 +122,7 @@ function createDefaultCameraControls(context: IContext, cam?: ICamera) {
         orbit.autoRotate = autoRotate != "0" && autoRotate?.toLowerCase() != "false";
         const autoRotateSpeed = Number.parseFloat(autoRotate || ".5");
         orbit.autoRotateSpeed = !isNaN(autoRotateSpeed) ? autoRotateSpeed : .5;
-        console.log("Auto-rotate", orbit.autoRotate, "speed:", orbit.autoRotateSpeed);
+        if(debug) console.log("Auto-rotate", orbit.autoRotate, "speed:", orbit.autoRotateSpeed);
         const autoFit = context.domElement.getAttribute("auto-fit");
         orbit.autoFit = autoFit !== "0" && autoFit?.toLowerCase() != "false";
         orbit.autoTarget = true;

package/src/engine-components/OrbitControls.ts

@@ -993,11 +993,50 @@ export class OrbitControls extends Behaviour implements ICameraController {
 
     // Adapted from https://discourse.threejs.org/t/camera-zoom-to-fit-object/936/24
     // Slower but better implementation that takes bones and exact vertex positions into account: https://github.com/google/model-viewer/blob/04e900c5027de8c5306fe1fe9627707f42811b05/packages/model-viewer/src/three-components/ModelScene.ts#L321
+
     /** 
      * Fits the camera to show the objects provided (defaults to the scene if no objects are passed in) 
+     * @param options The options for fitting the camera. Use to provide objects to fit to, fit direction and size and other settings.
     */
-    fitCamera(options?: OrbitFitCameraOptions): void {
+    fitCamera(options?: OrbitFitCameraOptions);
+    /** @deprecated Use fitCamera(options) */
+    fitCamera(objects?: Object3D | Array<Object3D>, options?: Omit<OrbitFitCameraOptions, "objects">);    
+    fitCamera(objectsOrOptions?: Object3D | Array<Object3D> | OrbitFitCameraOptions, options?: OrbitFitCameraOptions): void {
+
+
+        let objects: Object3D | Array<Object3D> | undefined = undefined;
+        // If the user passed in an array as first argument
+        if (Array.isArray(objectsOrOptions)) {
+            objects = objectsOrOptions;
+        }
+        // If the user passed in an object as first argument
+        else if (objectsOrOptions && "type" in objectsOrOptions) {
+            objects = objectsOrOptions;
+        }
+        // If the user passed in an object as first argument and options as second argument
+        else if (objectsOrOptions && typeof objectsOrOptions === "object") {
+            if (!(objectsOrOptions instanceof Object3D) && !Array.isArray(objectsOrOptions)) {
+                options = objectsOrOptions;
+                objects = options.objects;
+            }
+        }
+        // Ensure objects are setup correctly
+        if (objects && !Array.isArray(objects)) {
+            objects = [objects];
+        }
+        if (!Array.isArray(objects) || objects && objects.length <= 0) {
+            objects = this.context.scene.children;
+        }
+
+        // Make sure there's anything to fit to
+        if (!Array.isArray(objects) || objects.length <= 0) {
+            console.warn("No objects to fit camera to...");
+            return;
+        }
+
+
         const res = fitCamera({
+            objects: [...objects],
             ...options,
             autoApply: false,
             context: this.context,

package/src/engine-components/Renderer.ts

@@ -358,8 +358,11 @@ export class Renderer extends Behaviour implements IRenderer {
     //@ts-ignore
     get sharedMaterials(): SharedMaterialArray {
 
+        // @ts-ignore (original materials will be set during deserialization)
+        if(this._originalMaterials === undefined) return null;
+
         // @ts-ignore during deserialization code might access this property *before* the setter and then create an empty array
-        if (!this.__didAwake) return null;
+        if (this.__isDeserializing === true) return null;
 
         if (!this._sharedMaterials || !this._sharedMaterials.is(this)) {
             if (!this._originalMaterials) this._originalMaterials = [];
@@ -725,6 +728,8 @@ export class Renderer extends Behaviour implements IRenderer {
                 // If the material has a envMap and is NOT using a reflection probe we set the envMap to the scene environment
                 if (mat && "envMap" in mat && "envMapIntensity" in mat && !ReflectionProbe.isUsingReflectionProbe(mat)) {
                     mat.envMap = this.context.scene.environment;
+                    mat.envMapIntensity = this.context.scene.environmentIntensity;
+                    mat.envMapRotation = this.context.scene.environmentRotation;
                 }
             }
         }

package/src/engine-components/Skybox.ts

@@ -8,7 +8,7 @@ import { syncField } from "../engine/engine_networking_auto.js";
 import { loadPMREM } from "../engine/engine_pmrem.js";
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { type IContext } from "../engine/engine_types.js";
-import { addAttributeChangeCallback, getParam, PromiseAllWithErrors, removeAttributeChangeCallback } from "../engine/engine_utils.js";
+import { addAttributeChangeCallback, getParam, PromiseAllWithErrors, removeAttributeChangeCallback, toSourceId } from "../engine/engine_utils.js";
 import { registerObservableAttribute } from "../engine/webcomponents/needle-engine.extras.js";
 import { Camera, ClearFlags } from "./Camera.js";
 import { Behaviour, GameObject } from "./Component.js";
@@ -27,15 +27,34 @@ function createRemoteSkyboxComponent(context: IContext, url: string, skybox: boo
     }
 
     const remote = new RemoteSkybox();
+    remote.sourceId = toSourceId(url);
     remote.allowDrop = false;
     remote.allowNetworking = false;
     remote.background = skybox;
     remote.environment = environment;
     GameObject.addComponent(context.scene, remote);
     const urlChanged = newValue => {
-        if (typeof newValue !== "string") return;
-        if (debug) console.log(attribute, "CHANGED TO", newValue)
-        remote.setSkybox(newValue);
+        if (debug) console.log(attribute, "CHANGED TO", newValue);
+        if (newValue) {
+            if (typeof newValue !== "string") {
+                console.warn("Invalid attribute value for " + attribute);
+                return;
+            }
+            remote.setSkybox(newValue);
+        }
+        else {
+            if (remote.sourceId) {
+                if (environment) {
+                    if (!context.sceneLighting.internalEnableReflection(remote.sourceId)) {
+                        context.scene.environment = null;
+                    }
+                }
+                if (skybox) {
+                    const skybox = context.lightmaps.tryGetSkybox(remote.sourceId);
+                    context.scene.background = skybox;
+                }
+            }
+        }
     };
     addAttributeChangeCallback(context.domElement, attribute, urlChanged);
     remote.addEventListener("destroy", () => {
@@ -50,7 +69,7 @@ ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, (args) => {
     const context = args.context;
     const backgroundImage = context.domElement.getAttribute("background-image");
     const environmentImage = context.domElement.getAttribute("environment-image");
-    
+
     if (backgroundImage) {
         if (debug) console.log("Creating RemoteSkybox to load background " + backgroundImage);
         // if the user is loading a GLB without a camera then the CameraUtils (which creates the default camera)
@@ -246,8 +265,8 @@ export class RemoteSkybox extends Behaviour {
             envMap.needsUpdate = true;
         }
 
-        if(this.destroyed) return;
-        if(!this.context) {
+        if (this.destroyed) return;
+        if (!this.context) {
             console.warn("RemoteSkybox: Context is not available - can not apply skybox.");
             return;
         }

package/src/engine-components/codegen/components.ts

@@ -208,6 +208,7 @@ export { ClickThrough } from "../web/Clickthrough.js";
 export { CursorFollow } from "../web/CursorFollow.js";
 export { HoverAnimation } from "../web/HoverAnimation.js";
 export { ScrollFollow } from "../web/ScrollFollow.js";
+export { ViewBox } from "../web/ViewBox.js";
 export { Avatar } from "../webxr/Avatar.js";
 export { XRControllerFollow } from "../webxr/controllers/XRControllerFollow.js";
 export { XRControllerModel } from "../webxr/controllers/XRControllerModel.js";

package/src/engine-components/debug/LogStats.ts

@@ -4,6 +4,7 @@ import { Behaviour } from "../../engine-components/Component.js";
 
 const debug = getParam("logstats");
 
+/** @internal */
 export class LogStats extends Behaviour {
 
     onEnable(): void {

package/src/engine-components/timeline/PlayableDirector.ts

@@ -66,10 +66,19 @@ export class PlayableDirector extends Behaviour {
         this.createTrackFunctions[type] = fn;
     }
 
+    /**
+     * The timeline asset that is played by this director.
+     */
     playableAsset?: Models.TimelineAssetModel;
+
     /** Set to true to start playing the timeline when the scene starts */
     @serializable()
     playOnAwake?: boolean;
+
+    /**
+     * Determines how the timeline behaves when it reaches the end of its duration.
+     * @default DirectorWrapMode.Loop
+     */
     @serializable()
     extrapolationMode: DirectorWrapMode = DirectorWrapMode.Loop;
 
@@ -106,7 +115,7 @@ export class PlayableDirector extends Behaviour {
     /** @internal */
     awake(): void {
         if (debug)
-            console.log(this, this.playableAsset?.tracks);
+            console.log(this, this.playableAsset);
 
         this.rebuildGraph();
 

package/src/engine-components/timeline/TimelineModels.ts

@@ -92,4 +92,12 @@ export declare class SignalMarkerModel extends MarkerModel {
     asset: string;
 }
 
-export type ScrollMarkerModel = MarkerModel & { selector: string };
+/**
+ * Marker with a name, used for scroll-driven timelines. It is used together with elements in your HTML to define what time in the timeline should be active when the element is in the scroll view.
+ * 
+ * @example Mark html elements to define scroll positions 
+ * ```html
+ * <div data-timeline-marker>...</div>
+ * ```
+ */
+export type ScrollMarkerModel = MarkerModel & { name: string };

package/src/engine-components/timeline/TimelineTracks.ts

@@ -175,8 +175,10 @@ export class AnimationTrackHandler extends TrackHandler {
     // which means we want to notify the object that it's not animated anymore
     // and the animator can then take over
     onStateChanged() {
-        if (this._animator)
-            setObjectAnimated(this._animator.gameObject, this, this.director.isPlaying);
+        if (this._animator) {
+            // We can not check the *isPlaying* state here because the timeline might be paused and evaluated by e.g. ScrollFollow
+            setObjectAnimated(this._animator.gameObject, this, this.director.enabled && this.director.weight > 0);
+        }
     }
 
     createHooks(clipModel: Models.AnimationClipModel, clip) {
@@ -185,21 +187,22 @@ export class AnimationTrackHandler extends TrackHandler {
             return;
         }
         // we only want to hook into the binding of the root object
-        // TODO: test with a clip with multiple roots
-        const parts = clip.tracks[0].name.split(".");
-        const rootName = parts[parts.length - 2];
-        const positionTrackName = rootName + ".position";
-        const rotationTrackName = rootName + ".quaternion";
         let foundPositionTrack: boolean = false;
         let foundRotationTrack: boolean = false;
-        for (const t of clip.tracks) {
-            if (t.name.endsWith(positionTrackName)) {
-                foundPositionTrack = true;
-                this.createPositionInterpolant(clip, clipModel, t);
-            }
-            else if (t.name.endsWith(rotationTrackName)) {
-                foundRotationTrack = true;
-                this.createRotationInterpolant(clip, clipModel, t);
+        const parts = clip.tracks.find(t => t.name.includes(".position") || t.name.includes(".quaternion"))?.name.split(".");
+        if (parts) {
+            const rootName = parts[parts.length - 2];
+            const positionTrackName = rootName + ".position";
+            const rotationTrackName = rootName + ".quaternion";
+            for (const t of clip.tracks) {
+                if (!foundPositionTrack && t.name.endsWith(positionTrackName)) {
+                    foundPositionTrack = true;
+                    this.createPositionInterpolant(clip, clipModel, t);
+                }
+                else if (!foundRotationTrack && t.name.endsWith(rotationTrackName)) {
+                    foundRotationTrack = true;
+                    this.createRotationInterpolant(clip, clipModel, t);
+                }
             }
         }
 
@@ -224,16 +227,14 @@ export class AnimationTrackHandler extends TrackHandler {
                     const trackName = baseName + ".position";
                     if (debug) console.warn("Create position track", objName, targetObj);
                     // apply initial local position so it doesnt get flipped or otherwise changed
-                    const pos = targetObj.position;
-                    const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [pos.x, pos.y, pos.z, pos.x, pos.y, pos.z]);
+                    const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 0, 0, 0]);
                     clip.tracks.push(track);
                     this.createPositionInterpolant(clip, clipModel, track);
                 }
                 else if (!foundRotationTrack) {
                     const trackName = clip.tracks[0].name.substring(0, indexOfProperty) + ".quaternion";
                     if (debug) console.warn("Create quaternion track", objName, targetObj);
-                    const rot = targetObj.quaternion;
-                    const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [rot.x, rot.y, rot.z, rot.w, rot.x, rot.y, rot.z, rot.w]);
+                    const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 1, 0, 0, 0, 1]);
                     clip.tracks.push(track);
                     this.createRotationInterpolant(clip, clipModel, track);
                 }
@@ -832,23 +833,27 @@ export class AudioTrackHandler extends TrackHandler {
 
 export class MarkerTrackHandler extends TrackHandler {
     models: Array<Models.MarkerModel & Record<string, any>> = [];
-    isDirty = true;
+    needsSorting = true;
 
     *foreachMarker<T>(type: string | null = null) {
+        if(this.needsSorting) this.sort();
         for (const model of this.models) {
             if (model && model.type === type) yield model as T;
         }
     }
 
     onEnable() {
-        this.isDirty = true;
+        this.needsSorting = true;
     }
 
     evaluate(_time: number) {
-        if (this.isDirty) {
-            this.isDirty = false;
-            this.models.sort((a, b) => a.time - b.time);
-        }
+        if (this.needsSorting) this.sort();
+    }
+
+    private sort() {
+        this.needsSorting = false;
+        this.models.sort((a, b) => a.time - b.time);
+
     }
 }
 

package/src/engine-components/utils/LookAt.ts

@@ -1,5 +1,6 @@
 import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
 
+import { isDevEnvironment } from "../../engine/debug/index.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { lookAtObject } from "../../engine/engine_three_utils.js";
 import { type UsdzBehaviour } from "../../engine-components/export/usdz/extensions/behavior/Behaviour.js";
@@ -42,7 +43,10 @@ export class LookAt extends Behaviour implements UsdzBehaviour {
     /** @internal */
     onBeforeRender(): void {
         let target: Object3D | null | undefined = this.target;
-        if (!target) target = this.context.mainCamera;
+        if (!target) {
+            target = this.context.mainCamera;
+            if (isDevEnvironment()) console.warn(`[LookAt] No target set on ${this.name}, using main camera as target.`);
+        }
         if (!target) return;
 
         let copyTargetRotation = this.copyTargetRotation;

package/src/engine-components/web/Clickthrough.ts

@@ -1,12 +1,21 @@
 import { NEPointerEvent } from "../../engine/engine_input.js";
 import { onStart } from "../../engine/engine_lifecycle_api.js";
+import { addAttributeChangeCallback } from "../../engine/engine_utils.js";
 import { Behaviour } from "../Component.js";
 
 // Automatically add ClickThrough component if "clickthrough" attribute is present on the needle-engine element
 onStart(ctx => {
     const attribute = ctx.domElement.getAttribute("clickthrough");
-    if (attribute !== null && attribute !== "0" && attribute !== "false") {
-        ctx.scene.addComponent(ClickThrough);
+    if (clickthroughEnabled(attribute)) {
+        const comp = ctx.scene.addComponent(ClickThrough);
+        addAttributeChangeCallback(ctx.domElement, "clickthrough", () => {
+            const attribute = ctx.domElement.getAttribute("clickthrough");
+            comp.enabled = clickthroughEnabled(attribute);
+        });
+    }
+
+    function clickthroughEnabled(val: string | null) {
+        return val !== null && val !== "0" && val !== "false";
     }
 });