@needle-tools/engine 4.10.0-next.4f9d92a → 4.10.0-next.870425c

package/src/engine-components/Skybox.ts

@@ -8,7 +8,7 @@ import { syncField } from "../engine/engine_networking_auto.js";
 import { loadPMREM } from "../engine/engine_pmrem.js";
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { type IContext } from "../engine/engine_types.js";
-import { addAttributeChangeCallback, getParam, PromiseAllWithErrors, removeAttributeChangeCallback } from "../engine/engine_utils.js";
+import { addAttributeChangeCallback, getParam, PromiseAllWithErrors, removeAttributeChangeCallback, toSourceId } from "../engine/engine_utils.js";
 import { registerObservableAttribute } from "../engine/webcomponents/needle-engine.extras.js";
 import { Camera, ClearFlags } from "./Camera.js";
 import { Behaviour, GameObject } from "./Component.js";
@@ -27,15 +27,34 @@ function createRemoteSkyboxComponent(context: IContext, url: string, skybox: boo
     }
 
     const remote = new RemoteSkybox();
+    remote.sourceId = toSourceId(url);
     remote.allowDrop = false;
     remote.allowNetworking = false;
     remote.background = skybox;
     remote.environment = environment;
     GameObject.addComponent(context.scene, remote);
     const urlChanged = newValue => {
-        if (typeof newValue !== "string") return;
-        if (debug) console.log(attribute, "CHANGED TO", newValue)
-        remote.setSkybox(newValue);
+        if (debug) console.log(attribute, "CHANGED TO", newValue);
+        if (newValue) {
+            if (typeof newValue !== "string") {
+                console.warn("Invalid attribute value for " + attribute);
+                return;
+            }
+            remote.setSkybox(newValue);
+        }
+        else {
+            if (remote.sourceId) {
+                if (environment) {
+                    if (!context.sceneLighting.internalEnableReflection(remote.sourceId)) {
+                        context.scene.environment = null;
+                    }
+                }
+                if (skybox) {
+                    const skybox = context.lightmaps.tryGetSkybox(remote.sourceId);
+                    context.scene.background = skybox;
+                }
+            }
+        }
     };
     addAttributeChangeCallback(context.domElement, attribute, urlChanged);
     remote.addEventListener("destroy", () => {
@@ -50,7 +69,7 @@ ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, (args) => {
     const context = args.context;
     const backgroundImage = context.domElement.getAttribute("background-image");
     const environmentImage = context.domElement.getAttribute("environment-image");
-    
+
     if (backgroundImage) {
         if (debug) console.log("Creating RemoteSkybox to load background " + backgroundImage);
         // if the user is loading a GLB without a camera then the CameraUtils (which creates the default camera)
@@ -246,8 +265,8 @@ export class RemoteSkybox extends Behaviour {
             envMap.needsUpdate = true;
         }
 
-        if(this.destroyed) return;
-        if(!this.context) {
+        if (this.destroyed) return;
+        if (!this.context) {
             console.warn("RemoteSkybox: Context is not available - can not apply skybox.");
             return;
         }

package/src/engine-components/debug/LogStats.ts

@@ -4,6 +4,7 @@ import { Behaviour } from "../../engine-components/Component.js";
 
 const debug = getParam("logstats");
 
+/** @internal */
 export class LogStats extends Behaviour {
 
     onEnable(): void {

package/src/engine-components/timeline/PlayableDirector.ts

@@ -115,7 +115,7 @@ export class PlayableDirector extends Behaviour {
     /** @internal */
     awake(): void {
         if (debug)
-            console.log(this, this.playableAsset?.tracks);
+            console.log(this, this.playableAsset);
 
         this.rebuildGraph();
 

package/src/engine-components/timeline/TimelineModels.ts

@@ -1,11 +1,16 @@
 import { AnimationClip, Object3D, Quaternion, Vector3 } from "three";
-import { Behavior } from "three-mesh-ui";
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type TimelineAssetModel = {
     name: string;
     tracks: TrackModel[];
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export enum TrackType {
     Activation = "ActivationTrack",
     Animation = "AnimationTrack",
@@ -15,6 +20,9 @@ export enum TrackType {
     Signal = "SignalTrack",
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export enum ClipExtrapolation {
     None = 0,
     Hold = 1,
@@ -23,6 +31,9 @@ export enum ClipExtrapolation {
     Continue = 4
 };
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type TrackModel = {
     name: string;
     type: TrackType;
@@ -37,11 +48,17 @@ export declare type TrackModel = {
 declare type Vec3 = { x: number, y: number, z: number };
 declare type Quat = { x: number, y: number, z: number, w: number };
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type TrackOffset = {
     position: Vec3 | Vector3;
     rotation: Quat | Quaternion;
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type ClipModel = {
     start: number;
     end: number;
@@ -56,6 +73,9 @@ export declare type ClipModel = {
     reversed?: boolean;
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type AnimationClipModel = {
     clip: string | number | AnimationClip;
     loop: boolean;
@@ -65,12 +85,18 @@ export declare type AnimationClipModel = {
     rotation?: Quat | Quaternion;
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type AudioClipModel = {
     clip: string;
     loop: boolean;
     volume: number;
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare type ControlClipModel = {
     sourceObject: string | Object3D;
     controlActivation: boolean;
@@ -81,11 +107,16 @@ export enum MarkerType {
     Signal = "SignalEmitter",
 }
 
-export declare class MarkerModel {
+/**
+ * @category Animation and Sequencing
+ */export declare class MarkerModel {
     type: MarkerType;
     time: number;
 }
 
+/**
+ * @category Animation and Sequencing
+ */
 export declare class SignalMarkerModel extends MarkerModel {
     retroActive: boolean;
     emitOnce: boolean;
@@ -99,5 +130,8 @@ export declare class SignalMarkerModel extends MarkerModel {
  * ```html
  * <div data-timeline-marker>...</div>
  * ```
- */
+ * 
+ * @link [Example Project using ScrollMarker](https://scrollytelling-bike-z23hmxb2gnu5a.needle.run/)
+ * @category Animation and Sequencing
+*/
 export type ScrollMarkerModel = MarkerModel & { name: string };

package/src/engine-components/timeline/TimelineTracks.ts

@@ -187,21 +187,22 @@ export class AnimationTrackHandler extends TrackHandler {
             return;
         }
         // we only want to hook into the binding of the root object
-        // TODO: test with a clip with multiple roots
-        const parts = clip.tracks[0].name.split(".");
-        const rootName = parts[parts.length - 2];
-        const positionTrackName = rootName + ".position";
-        const rotationTrackName = rootName + ".quaternion";
         let foundPositionTrack: boolean = false;
         let foundRotationTrack: boolean = false;
-        for (const t of clip.tracks) {
-            if (t.name.endsWith(positionTrackName)) {
-                foundPositionTrack = true;
-                this.createPositionInterpolant(clip, clipModel, t);
-            }
-            else if (t.name.endsWith(rotationTrackName)) {
-                foundRotationTrack = true;
-                this.createRotationInterpolant(clip, clipModel, t);
+        const parts = clip.tracks.find(t => t.name.includes(".position") || t.name.includes(".quaternion"))?.name.split(".");
+        if (parts) {
+            const rootName = parts[parts.length - 2];
+            const positionTrackName = rootName + ".position";
+            const rotationTrackName = rootName + ".quaternion";
+            for (const t of clip.tracks) {
+                if (!foundPositionTrack && t.name.endsWith(positionTrackName)) {
+                    foundPositionTrack = true;
+                    this.createPositionInterpolant(clip, clipModel, t);
+                }
+                else if (!foundRotationTrack && t.name.endsWith(rotationTrackName)) {
+                    foundRotationTrack = true;
+                    this.createRotationInterpolant(clip, clipModel, t);
+                }
             }
         }
 
@@ -226,16 +227,14 @@ export class AnimationTrackHandler extends TrackHandler {
                     const trackName = baseName + ".position";
                     if (debug) console.warn("Create position track", objName, targetObj);
                     // apply initial local position so it doesnt get flipped or otherwise changed
-                    const pos = targetObj.position;
-                    const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [pos.x, pos.y, pos.z, pos.x, pos.y, pos.z]);
+                    const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 0, 0, 0]);
                     clip.tracks.push(track);
                     this.createPositionInterpolant(clip, clipModel, track);
                 }
                 else if (!foundRotationTrack) {
                     const trackName = clip.tracks[0].name.substring(0, indexOfProperty) + ".quaternion";
                     if (debug) console.warn("Create quaternion track", objName, targetObj);
-                    const rot = targetObj.quaternion;
-                    const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [rot.x, rot.y, rot.z, rot.w, rot.x, rot.y, rot.z, rot.w]);
+                    const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 1, 0, 0, 0, 1]);
                     clip.tracks.push(track);
                     this.createRotationInterpolant(clip, clipModel, track);
                 }
@@ -834,23 +833,27 @@ export class AudioTrackHandler extends TrackHandler {
 
 export class MarkerTrackHandler extends TrackHandler {
     models: Array<Models.MarkerModel & Record<string, any>> = [];
-    isDirty = true;
+    needsSorting = true;
 
     *foreachMarker<T>(type: string | null = null) {
+        if(this.needsSorting) this.sort();
         for (const model of this.models) {
             if (model && model.type === type) yield model as T;
         }
     }
 
     onEnable() {
-        this.isDirty = true;
+        this.needsSorting = true;
     }
 
     evaluate(_time: number) {
-        if (this.isDirty) {
-            this.isDirty = false;
-            this.models.sort((a, b) => a.time - b.time);
-        }
+        if (this.needsSorting) this.sort();
+    }
+
+    private sort() {
+        this.needsSorting = false;
+        this.models.sort((a, b) => a.time - b.time);
+
     }
 }
 

package/src/engine-components/web/ScrollFollow.ts

@@ -1,10 +1,12 @@
+// For firefox ViewTimeline support
+import "scroll-timeline-polyfill/dist/scroll-timeline.js";
+
 import { Box3, Object3D } from "three";
-import { element } from "three/src/nodes/TSL.js";
 
-import { Context } from "../../engine/engine_context.js";
+import { isDevEnvironment } from "../../engine/debug/debug.js";
 import { Mathf } from "../../engine/engine_math.js";
 import { serializable } from "../../engine/engine_serialization.js";
-import { getBoundingBox } from "../../engine/engine_three_utils.js";
+import { getBoundingBox, setVisibleInCustomShadowRendering } from "../../engine/engine_three_utils.js";
 import { getParam } from "../../engine/engine_utils.js";
 import { Animation } from "../Animation.js";
 import { Animator } from "../Animator.js";
@@ -38,6 +40,7 @@ type ScrollFollowEvent = {
  * 
  * @link Example at https://scrollytelling-2-z23hmxby7c6x-u30ld.needle.run/
  * @link Template at https://github.com/needle-engine/scrollytelling-template
+ * @link [Scrollytelling Bike Demo](https://scrollytelling-bike-z23hmxb2gnu5a.needle.run/)   
  * 
  * ## How to use with an Animator
  * 1. Create an Animator component and set up a float parameter named "scroll".
@@ -173,8 +176,8 @@ export class ScrollFollow extends Behaviour {
 
                 const value = this.invert ? 1 - this._current_value : this._current_value;
 
-                const height = this._rangeEndValue - this._rangeStartValue;
-                const pixelValue = this._rangeStartValue + value * height;
+                // const height = this._rangeEndValue - this._rangeStartValue;
+                // const pixelValue = this._rangeStartValue + value * height;
 
                 // apply scroll to target(s)
                 if (Array.isArray(this.target)) {
@@ -185,7 +188,7 @@ export class ScrollFollow extends Behaviour {
                 }
 
                 if (debug && this.context.time.frame % 30 === 0) {
-                    console.debug(`[ScrollFollow] ${this._current_value.toFixed(5)} — ${(this._target_value * 100).toFixed(0)}%`);
+                    console.debug(`[ScrollFollow] ${this._current_value.toFixed(5)} — ${(this._target_value * 100).toFixed(0)}%, targets [${Array.isArray(this.target) ? this.target.length : 1}]`);
                 }
             }
         }
@@ -303,22 +306,24 @@ export class ScrollFollow extends Behaviour {
             const index = markerIndex++;
 
             // Get marker elements from DOM
-            if ((marker.element === undefined || marker.needsUpdate === true || /** element is not in DOM anymore? */ (!marker.element?.parentNode))) {
+            if ((marker.element === undefined || marker.needsUpdate === true || /** element is not in DOM anymore? */ (marker.element && !marker.element?.parentNode))) {
                 marker.needsUpdate = false;
                 try {
+                    // TODO: with this it's currently not possible to remap markers from HTML. For example if I have two sections and I want to now use the marker["center"] multiple times to stay at that marker for a longer time
                     marker.element = tryGetElementsForSelector(index, marker.name) as HTMLElement | null;
-                    if (debug) console.debug("ScrollMarker found on page", marker.element, marker.name);
-                    // if (!marker.element) {
-                    //     marker.timeline = undefined;
-                    //     continue;
-                    // }
-                    // else {
-                    //     /** @ts-ignore */
-                    //     marker.timeline = new ViewTimeline({
-                    //         subject: marker.element,
-                    //         axis: 'block', // https://drafts.csswg.org/scroll-animations/#scroll-notation
-                    //     });
-                    // }
+                    if (debug) console.debug(`ScrollMarker #${index} "${marker.name}" (${marker.time.toFixed(2)}) found`, marker.element);
+                    if (!marker.element) {
+                        marker.timeline = undefined;
+                        if (debug || isDevEnvironment()) console.warn(`No HTML element found for ScrollMarker: ${marker.name} (index ${index})`);
+                        continue;
+                    }
+                    else {
+                        /** @ts-ignore */
+                        marker.timeline = new ViewTimeline({
+                            subject: marker.element,
+                            axis: 'block', // https://drafts.csswg.org/scroll-animations/#scroll-notation
+                        });
+                    }
                 }
                 catch (error) {
                     marker.element = null;
@@ -349,57 +354,75 @@ export class ScrollFollow extends Behaviour {
 
         weightsArray.length = 0;
         let sum = 0;
+        const oneFrameTime = 1 / 60;
 
         // We keep a separate count here in case there are some markers that could not be resolved so point to *invalid* elements - the timeline should fallback to 0-1 scroll behaviour then
         let markerCount = 0;
-        for (const marker of markersArray) {
-
+        for (let i = 0; i < markersArray.length; i++) {
+            const marker = markersArray[i];
             if (!marker.element) continue;
+            const nextMarker = markersArray[i + 1];
+
+            const nextTime = nextMarker
+                ? (nextMarker.time - oneFrameTime)
+                : duration;
 
             markerCount += 1;
 
-            // const timeline = marker.timeline;
-            // if (timeline) {
-            //     const time01 = calculateTimelinePositionNormalized(timeline);
-            //     if (time01 > 0 && time01 <= 1) {
-            //         const overlap = calculateTimelinePositionNormalized(timeline!);
-            //         const weight = overlap;
-            //         // console.log(marker.element.className, time01)
-            //         weightsArray.push({ time: marker.time, weight: weight });
-            //         sum += weight;
-            //     }
-            // }
-            // continue;
+            const timeline = marker.timeline;
+            if (timeline) {
+                const time01 = calculateTimelinePositionNormalized(timeline);
+                // remap 0-1 to 0 - 1 - 0 (full weight at center)
+                const weight = 1 - Math.abs(time01 - 0.5) * 2;
+                const name = marker.name || `marker${i}`;
+                if (time01 > 0 && time01 <= 1) {
+                    const lerpTime = marker.time + (nextTime - marker.time) * time01;
+                    weightsArray.push({ name, time: lerpTime, weight: weight });
+                    sum += weight;
+                }
+                // Before the first marker is reached
+                else if (i === 0 && time01 <= 0) {
+                    weightsArray.push({ name, time: 0, weight: 1 });
+                    sum += 1;
+                }
+                // After the last marker is reached
+                else if (i === markersArray.length - 1 && time01 >= 1) {
+                    weightsArray.push({ name, time: duration, weight: 1 });
+                    sum += 1;
+                }
+            }
+            continue;
             // if(this.context.time.frame % 10 === 0) console.log(marker.element?.className, timeline, calculateTimelinePositionNormalized(timeline!));
 
-            const top = marker.element.offsetTop;
-            const height = marker.element.offsetHeight;
-            const bottom = top + height;
-            let overlap = 0;
+            // const top = marker.element.offsetTop - this._scrollContainerHeight;
+            // const height = marker.element.offsetHeight + this._scrollContainerHeight;
+            // const bottom = top + height;
+            // let overlap = 0;
 
-            // TODO: if we have two marker sections where no HTML overlaps (vor example because some large section is between them) we probably want to still virtually interpolate between them slowly in that region
+            // // TODO: if we have two marker sections where no HTML overlaps (vor example because some large section is between them) we probably want to still virtually interpolate between them slowly in that region
 
-            if (bottom < currentTop) {
-                // marker is above scroll region
-                overlap = 0;
-            }
-            else if (top > currentBottom) {
-                // marker is below scroll region
-                overlap = 0;
-            }
-            else {
-                // calculate overlap in pixels
-                const overlapTop = Math.max(top, currentTop);
-                const overlapBottom = Math.min(bottom, currentBottom);
-                overlap = Math.max(0, overlapBottom - overlapTop);
-            }
+            // if (bottom < currentTop) {
+            //     // marker is above scroll region
+            //     overlap = 0;
+            // }
+            // else if (top > currentBottom) {
+            //     // marker is below scroll region
+            //     overlap = 0;
+            // }
+            // else {
+            //     // calculate overlap in pixels
+            //     const overlapTop = Math.max(top, currentTop);
+            //     const overlapBottom = Math.min(bottom, currentBottom);
+            //     const height = Math.max(1, currentBottom - currentTop);
+            //     overlap = Math.max(0, overlapBottom - overlapTop);
+            // }
 
-            markerCount += 1;
+            // // if(this.context.time.frame % 20 === 0) console.log(overlap)
 
-            if (overlap > 0) {
-                weightsArray.push({ time: marker.time, weight: overlap });
-                sum += overlap;
-            }
+            // if (overlap > 0) {
+            //     weightsArray.push({ time: marker.time, weight: overlap });
+            //     sum += overlap;
+            // }
         }
 
         if (weightsArray.length <= 0 && markerCount <= 0) {
@@ -407,19 +430,26 @@ export class ScrollFollow extends Behaviour {
         }
         else if (weightsArray.length > 0) {
             // normalize and calculate weighted time
-            let time = 0;
-            for (const entry of weightsArray) {
-                const weight = entry.weight / Math.max(0.00001, sum);
-                // console.log(weight.toFixed(2))
-                // lerp time based on weight
-                const diff = Math.abs(entry.time - time);
-                time += diff * weight;
+            let time = weightsArray[0].time; // fallback to first time
+            if (weightsArray.length > 1) {
+                for (const entry of weightsArray) {
+                    const weight = entry.weight / Math.max(0.00001, sum);
+                    // console.log(weight.toFixed(2))
+                    // lerp time based on weight
+                    const diff = Math.abs(entry.time - time);
+                    time += diff * weight;
+                }
             }
-            // console.log(time.toFixed(2), [...weightsArray])
-            if (this.damping <= 0)
+            if (this.damping <= 0) {
                 director.time = time;
-            else
+            }
+            else {
                 director.time = Mathf.lerp(director.time, time, this.context.time.deltaTime / this.damping);
+            }
+
+            if (debug && this.context.time.frame % 30 === 0) {
+                console.log(`[ScrollFollow ] Timeline ${director.name}: ${time.toFixed(3)}`, weightsArray.map(w => `[${w.name} ${(w.weight * 100).toFixed(0)}%]`).join(", "));
+            }
         }
     }
 
@@ -428,9 +458,13 @@ export class ScrollFollow extends Behaviour {
 
 
 const weightsArray: OverlapInfo[] = [];
-const markersArray: (ScrollMarkerModel & { element?: HTMLElement | null, timeline?: ViewTimeline })[] = [];
+const markersArray: Array<ScrollMarkerModel & {
+    element?: HTMLElement | null,
+    timeline?: ViewTimeline,
+}> = [];
 
 type OverlapInfo = {
+    name: string,
     /** Marker time */
     time: number,
     /** Overlap in pixels */
@@ -446,17 +480,29 @@ type OverlapInfo = {
 // }
 // const querySelectorResults: Array<SelectorCache> = [];
 
-const needleScrollMarkerCacheKey = "data-timeline-marker";
 const needleScrollMarkerIndexCache = new Map<number, Element | null>();
 const needleScrollMarkerNameCache = new Map<string, Element | null>();
 let needsScrollMarkerRefresh = true;
 
-function tryGetElementsForSelector(index: number, name: string): Element | null {
+function tryGetElementsForSelector(index: number, name: string, _cycle: number = 0): Element | null {
 
     if (!needsScrollMarkerRefresh) {
-        let element = name?.length ? needleScrollMarkerNameCache.get(name) : null;
-        if (element) return element;
-        element = needleScrollMarkerIndexCache.get(index) || null;
+        if (name?.length) {
+            const element = needleScrollMarkerNameCache.get(name) || null;
+            if (element) return element;
+            // const isNumber = !isNaN(Number(name));
+            // if (!isNumber) {
+            // }
+        }
+        const element = needleScrollMarkerIndexCache.get(index) || null;
+        const value = element?.getAttribute("data-timeline-marker");
+        // if (value?.length) {
+        //     if (cycle === 0) {
+        //         // if the HTML marker we found by index does define a different marker name we try to find the correct HTML element by name
+        //         return tryGetElementsForSelector(index, value, 1);
+        //     }
+        //     if (isDevEnvironment()) console.warn(`ScrollMarker name mismatch: expected "${name}", got "${value}"`);
+        // }
         return element;
     }
     needsScrollMarkerRefresh = false;
@@ -467,26 +513,8 @@ function tryGetElementsForSelector(index: number, name: string): Element | null
         const name = m.getAttribute("data-timeline-marker");
         if (name?.length) needleScrollMarkerNameCache.set(name, m);
     });
-    const element = needleScrollMarkerIndexCache.get(index) || null;
-    return element;
-
-
-    /* e.g. 
-    <div class="section behind start" data-needle-scroll-marker>
-    */
-    // console.log(index, element)
-    if (element) return element;
-
-    // for (const entry of querySelectorResults) {
-    //     if (entry.selector === selector) {
-    //         const index = entry.usedElementCount++;
-    //         return entry.elements && index < entry.elements.length ? entry.elements[index] : null;
-    //     }
-    // }
-    // const elements = document.querySelectorAll(selector);
-    // querySelectorResults.push({ selector, elements: Array.from(elements), usedElementCount: 1 });
-    // if (elements.length > 0) return elements[0];
-    return null;
+    needsScrollMarkerRefresh = false;
+    return tryGetElementsForSelector(index, name);
 }
 
 
@@ -506,16 +534,6 @@ function calculateTimelinePositionNormalized(timeline: ViewTimeline) {
     const t01 = currentTime.unit === "seconds" ? (currentTime.value / durationValue) : (currentTime.value / 100);
     return t01;
 }
-function calculateNormalizedOverlap(timeline: ViewTimeline) {
-    if (!timeline.source) return 0;
-    const start = timeline.startOffset;
-    const end = timeline.endOffset;
-    const total = start.value + end.value;
-    if (total <= 0) return 1;
-    const startNorm = start.value / total;
-    const endNorm = end.value / total;
-    return 1 - (startNorm + endNorm);
-}
 
 
 declare global {