@needle-tools/engine 4.10.0-next.f0ec242 → 4.10.0

package/src/engine-components/timeline/TimelineTracks.ts

@@ -175,8 +175,10 @@ export class AnimationTrackHandler extends TrackHandler {
     // which means we want to notify the object that it's not animated anymore
     // and the animator can then take over
     onStateChanged() {
-        if (this._animator)
-            setObjectAnimated(this._animator.gameObject, this, this.director.isPlaying);
+        if (this._animator) {
+            // We can not check the *isPlaying* state here because the timeline might be paused and evaluated by e.g. ScrollFollow
+            setObjectAnimated(this._animator.gameObject, this, this.director.enabled && this.director.weight > 0);
+        }
     }
 
     createHooks(clipModel: Models.AnimationClipModel, clip) {
@@ -185,21 +187,22 @@ export class AnimationTrackHandler extends TrackHandler {
             return;
         }
         // we only want to hook into the binding of the root object
-        // TODO: test with a clip with multiple roots
-        const parts = clip.tracks[0].name.split(".");
-        const rootName = parts[parts.length - 2];
-        const positionTrackName = rootName + ".position";
-        const rotationTrackName = rootName + ".quaternion";
         let foundPositionTrack: boolean = false;
         let foundRotationTrack: boolean = false;
-        for (const t of clip.tracks) {
-            if (t.name.endsWith(positionTrackName)) {
-                foundPositionTrack = true;
-                this.createPositionInterpolant(clip, clipModel, t);
-            }
-            else if (t.name.endsWith(rotationTrackName)) {
-                foundRotationTrack = true;
-                this.createRotationInterpolant(clip, clipModel, t);
+        const parts = clip.tracks.find(t => t.name.includes(".position") || t.name.includes(".quaternion"))?.name.split(".");
+        if (parts) {
+            const rootName = parts[parts.length - 2];
+            const positionTrackName = rootName + ".position";
+            const rotationTrackName = rootName + ".quaternion";
+            for (const t of clip.tracks) {
+                if (!foundPositionTrack && t.name.endsWith(positionTrackName)) {
+                    foundPositionTrack = true;
+                    this.createPositionInterpolant(clip, clipModel, t);
+                }
+                else if (!foundRotationTrack && t.name.endsWith(rotationTrackName)) {
+                    foundRotationTrack = true;
+                    this.createRotationInterpolant(clip, clipModel, t);
+                }
             }
         }
 
@@ -224,16 +227,14 @@ export class AnimationTrackHandler extends TrackHandler {
                     const trackName = baseName + ".position";
                     if (debug) console.warn("Create position track", objName, targetObj);
                     // apply initial local position so it doesnt get flipped or otherwise changed
-                    const pos = targetObj.position;
-                    const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [pos.x, pos.y, pos.z, pos.x, pos.y, pos.z]);
+                    const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 0, 0, 0]);
                     clip.tracks.push(track);
                     this.createPositionInterpolant(clip, clipModel, track);
                 }
                 else if (!foundRotationTrack) {
                     const trackName = clip.tracks[0].name.substring(0, indexOfProperty) + ".quaternion";
                     if (debug) console.warn("Create quaternion track", objName, targetObj);
-                    const rot = targetObj.quaternion;
-                    const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [rot.x, rot.y, rot.z, rot.w, rot.x, rot.y, rot.z, rot.w]);
+                    const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 1, 0, 0, 0, 1]);
                     clip.tracks.push(track);
                     this.createRotationInterpolant(clip, clipModel, track);
                 }
@@ -832,23 +833,27 @@ export class AudioTrackHandler extends TrackHandler {
 
 export class MarkerTrackHandler extends TrackHandler {
     models: Array<Models.MarkerModel & Record<string, any>> = [];
-    isDirty = true;
+    needsSorting = true;
 
     *foreachMarker<T>(type: string | null = null) {
+        if(this.needsSorting) this.sort();
         for (const model of this.models) {
             if (model && model.type === type) yield model as T;
         }
     }
 
     onEnable() {
-        this.isDirty = true;
+        this.needsSorting = true;
     }
 
     evaluate(_time: number) {
-        if (this.isDirty) {
-            this.isDirty = false;
-            this.models.sort((a, b) => a.time - b.time);
-        }
+        if (this.needsSorting) this.sort();
+    }
+
+    private sort() {
+        this.needsSorting = false;
+        this.models.sort((a, b) => a.time - b.time);
+
     }
 }
 

package/src/engine-components/utils/LookAt.ts

@@ -1,5 +1,6 @@
 import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
 
+import { isDevEnvironment } from "../../engine/debug/index.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { lookAtObject } from "../../engine/engine_three_utils.js";
 import { type UsdzBehaviour } from "../../engine-components/export/usdz/extensions/behavior/Behaviour.js";
@@ -42,7 +43,10 @@ export class LookAt extends Behaviour implements UsdzBehaviour {
     /** @internal */
     onBeforeRender(): void {
         let target: Object3D | null | undefined = this.target;
-        if (!target) target = this.context.mainCamera;
+        if (!target) {
+            target = this.context.mainCamera;
+            if (isDevEnvironment()) console.warn(`[LookAt] No target set on ${this.name}, using main camera as target.`);
+        }
         if (!target) return;
 
         let copyTargetRotation = this.copyTargetRotation;

package/src/engine-components/web/Clickthrough.ts

@@ -1,12 +1,21 @@
 import { NEPointerEvent } from "../../engine/engine_input.js";
 import { onStart } from "../../engine/engine_lifecycle_api.js";
+import { addAttributeChangeCallback } from "../../engine/engine_utils.js";
 import { Behaviour } from "../Component.js";
 
 // Automatically add ClickThrough component if "clickthrough" attribute is present on the needle-engine element
 onStart(ctx => {
     const attribute = ctx.domElement.getAttribute("clickthrough");
-    if (attribute !== null && attribute !== "0" && attribute !== "false") {
-        ctx.scene.addComponent(ClickThrough);
+    if (clickthroughEnabled(attribute)) {
+        const comp = ctx.scene.addComponent(ClickThrough);
+        addAttributeChangeCallback(ctx.domElement, "clickthrough", () => {
+            const attribute = ctx.domElement.getAttribute("clickthrough");
+            comp.enabled = clickthroughEnabled(attribute);
+        });
+    }
+
+    function clickthroughEnabled(val: string | null) {
+        return val !== null && val !== "0" && val !== "false";
     }
 });
 

package/src/engine-components/web/ScrollFollow.ts

@@ -1,10 +1,12 @@
+// For firefox ViewTimeline support
+import "scroll-timeline-polyfill/dist/scroll-timeline.js";
+
 import { Box3, Object3D } from "three";
-import { element } from "three/src/nodes/TSL.js";
 
-import { Context } from "../../engine/engine_context.js";
+import { isDevEnvironment } from "../../engine/debug/debug.js";
 import { Mathf } from "../../engine/engine_math.js";
 import { serializable } from "../../engine/engine_serialization.js";
-import { getBoundingBox } from "../../engine/engine_three_utils.js";
+import { getBoundingBox, setVisibleInCustomShadowRendering } from "../../engine/engine_three_utils.js";
 import { getParam } from "../../engine/engine_utils.js";
 import { Animation } from "../Animation.js";
 import { Animator } from "../Animator.js";
@@ -38,6 +40,7 @@ type ScrollFollowEvent = {
  * 
  * @link Example at https://scrollytelling-2-z23hmxby7c6x-u30ld.needle.run/
  * @link Template at https://github.com/needle-engine/scrollytelling-template
+ * @link [Scrollytelling Bike Demo](https://scrollytelling-bike-z23hmxb2gnu5a.needle.run/)   
  * 
  * ## How to use with an Animator
  * 1. Create an Animator component and set up a float parameter named "scroll".
@@ -150,7 +153,8 @@ export class ScrollFollow extends Behaviour {
             }
         }
 
-        if (this._current_value !== this._appliedValue) {
+        // if (this._current_value !== this._appliedValue) 
+        {
             this._appliedValue = this._current_value;
 
             let defaultPrevented = false;
@@ -172,8 +176,8 @@ export class ScrollFollow extends Behaviour {
 
                 const value = this.invert ? 1 - this._current_value : this._current_value;
 
-                const height = this._rangeEndValue - this._rangeStartValue;
-                const pixelValue = this._rangeStartValue + value * height;
+                // const height = this._rangeEndValue - this._rangeStartValue;
+                // const pixelValue = this._rangeStartValue + value * height;
 
                 // apply scroll to target(s)
                 if (Array.isArray(this.target)) {
@@ -184,7 +188,7 @@ export class ScrollFollow extends Behaviour {
                 }
 
                 if (debug && this.context.time.frame % 30 === 0) {
-                    console.debug(`[ScrollFollow] ${this._current_value.toFixed(5)} — ${(this._target_value * 100).toFixed(0)}%`);
+                    console.debug(`[ScrollFollow] ${this._current_value.toFixed(5)} — ${(this._target_value * 100).toFixed(0)}%, targets [${Array.isArray(this.target) ? this.target.length : 1}]`);
                 }
             }
         }
@@ -242,7 +246,8 @@ export class ScrollFollow extends Behaviour {
 
         if (target instanceof PlayableDirector) {
             this.handleTimelineTarget(target, value);
-            if (!target.isPlaying) target.evaluate();
+            if (target.isPlaying) target.pause();
+            target.evaluate();
         }
         else if (target instanceof Animator) {
             target.setFloat("scroll", value);
@@ -292,18 +297,37 @@ export class ScrollFollow extends Behaviour {
         let scrollRegionEnd = 0;
         markersArray.length = 0;
 
-        for (const marker of director.foreachMarker<ScrollMarkerModel & { element?: HTMLElement | null, needsUpdate?: boolean }>("ScrollMarker")) {
+        // querySelectorResults.length = 0;
+        let markerIndex = 0;
+
+        // https://scroll-driven-animations.style/tools/view-timeline/ranges
+        for (const marker of director.foreachMarker<ScrollMarkerModel & { element?: HTMLElement | null, needsUpdate?: boolean, timeline?: ViewTimeline }>("ScrollMarker")) {
+
+            const index = markerIndex++;
 
             // Get marker elements from DOM
-            if (marker.selector?.length && (marker.element === undefined || marker.needsUpdate === true || /** element is not in DOM anymore? */ (!marker.element?.parentNode))) {
+            if ((marker.element === undefined || marker.needsUpdate === true || /** element is not in DOM anymore? */ (marker.element && !marker.element?.parentNode))) {
                 marker.needsUpdate = false;
                 try {
-                    marker.element = document.querySelector<HTMLElement>(marker.selector) || null;
-                    if (debug) console.debug("ScrollMarker found on page", marker.element, marker.selector);
+                    // TODO: with this it's currently not possible to remap markers from HTML. For example if I have two sections and I want to now use the marker["center"] multiple times to stay at that marker for a longer time
+                    marker.element = tryGetElementsForSelector(index, marker.name) as HTMLElement | null;
+                    if (debug) console.debug(`ScrollMarker #${index} "${marker.name}" (${marker.time.toFixed(2)}) found`, marker.element);
+                    if (!marker.element) {
+                        marker.timeline = undefined;
+                        if (debug || isDevEnvironment()) console.warn(`No HTML element found for ScrollMarker: ${marker.name} (index ${index})`);
+                        continue;
+                    }
+                    else {
+                        /** @ts-ignore */
+                        marker.timeline = new ViewTimeline({
+                            subject: marker.element,
+                            axis: 'block', // https://drafts.csswg.org/scroll-animations/#scroll-notation
+                        });
+                    }
                 }
                 catch (error) {
                     marker.element = null;
-                    console.error("ScrollMarker selector is not valid: " + marker.selector + "\n", error);
+                    console.error("ScrollMarker selector is not valid: " + marker.name + "\n", error);
                 }
             }
 
@@ -330,65 +354,195 @@ export class ScrollFollow extends Behaviour {
 
         weightsArray.length = 0;
         let sum = 0;
+        const oneFrameTime = 1 / 60;
 
+        // We keep a separate count here in case there are some markers that could not be resolved so point to *invalid* elements - the timeline should fallback to 0-1 scroll behaviour then
         let markerCount = 0;
-        for (const marker of markersArray) {
-
+        for (let i = 0; i < markersArray.length; i++) {
+            const marker = markersArray[i];
             if (!marker.element) continue;
-
-            const top = marker.element.offsetTop;
-            const height = marker.element.offsetHeight;
-            const bottom = top + height;
-            let overlap = 0;
-
-            // TODO: if we have two marker sections where no HTML overlaps (vor example because some large section is between them) we probably want to still virtually interpolate between them slowly in that region
-
-            if (bottom < currentTop) {
-                // marker is above scroll region
-                overlap = 0;
-            }
-            else if (top > currentBottom) {
-                // marker is below scroll region
-                overlap = 0;
-            }
-            else {
-                // calculate overlap in pixels
-                const overlapTop = Math.max(top, currentTop);
-                const overlapBottom = Math.min(bottom, currentBottom);
-                overlap = Math.max(0, overlapBottom - overlapTop);
-                // console.log(marker.element.className, overlap)
-            }
-
-            if (overlap > 0) {
-                weightsArray.push({ time: marker.time, weight: overlap });
-                sum += overlap;
+            const nextMarker = markersArray[i + 1];
+
+            const nextTime = nextMarker
+                ? (nextMarker.time - oneFrameTime)
+                : duration;
+
+            markerCount += 1;
+
+            const timeline = marker.timeline;
+            if (timeline) {
+                const time01 = calculateTimelinePositionNormalized(timeline);
+                // remap 0-1 to 0 - 1 - 0 (full weight at center)
+                const weight = 1 - Math.abs(time01 - 0.5) * 2;
+                const name = marker.name || `marker${i}`;
+                if (time01 > 0 && time01 <= 1) {
+                    const lerpTime = marker.time + (nextTime - marker.time) * time01;
+                    weightsArray.push({ name, time: lerpTime, weight: weight });
+                    sum += weight;
+                }
+                // Before the first marker is reached
+                else if (i === 0 && time01 <= 0) {
+                    weightsArray.push({ name, time: 0, weight: 1 });
+                    sum += 1;
+                }
+                // After the last marker is reached
+                else if (i === markersArray.length - 1 && time01 >= 1) {
+                    weightsArray.push({ name, time: duration, weight: 1 });
+                    sum += 1;
+                }
             }
+            continue;
+            // if(this.context.time.frame % 10 === 0) console.log(marker.element?.className, timeline, calculateTimelinePositionNormalized(timeline!));
+
+            // const top = marker.element.offsetTop - this._scrollContainerHeight;
+            // const height = marker.element.offsetHeight + this._scrollContainerHeight;
+            // const bottom = top + height;
+            // let overlap = 0;
+
+            // // TODO: if we have two marker sections where no HTML overlaps (vor example because some large section is between them) we probably want to still virtually interpolate between them slowly in that region
+
+            // if (bottom < currentTop) {
+            //     // marker is above scroll region
+            //     overlap = 0;
+            // }
+            // else if (top > currentBottom) {
+            //     // marker is below scroll region
+            //     overlap = 0;
+            // }
+            // else {
+            //     // calculate overlap in pixels
+            //     const overlapTop = Math.max(top, currentTop);
+            //     const overlapBottom = Math.min(bottom, currentBottom);
+            //     const height = Math.max(1, currentBottom - currentTop);
+            //     overlap = Math.max(0, overlapBottom - overlapTop);
+            // }
+
+            // // if(this.context.time.frame % 20 === 0) console.log(overlap)
+
+            // if (overlap > 0) {
+            //     weightsArray.push({ time: marker.time, weight: overlap });
+            //     sum += overlap;
+            // }
         }
 
-        if (weightsArray.length <= 0 && markersArray.length <= 0) {
+        if (weightsArray.length <= 0 && markerCount <= 0) {
             director.time = value * duration;
         }
         else if (weightsArray.length > 0) {
             // normalize and calculate weighted time
-            let time = weightsArray[0].time;
-            for (const o of weightsArray) {
-                const weight = o.weight / Math.max(0.00001, sum);
-                // lerp time based on weight
-                const diff = Math.abs(o.time - time);
-                time += diff * weight;
+            let time = weightsArray[0].time; // fallback to first time
+            if (weightsArray.length > 1) {
+                for (const entry of weightsArray) {
+                    const weight = entry.weight / Math.max(0.00001, sum);
+                    // console.log(weight.toFixed(2))
+                    // lerp time based on weight
+                    const diff = Math.abs(entry.time - time);
+                    time += diff * weight;
+                }
+            }
+            if (this.damping <= 0) {
+                director.time = time;
+            }
+            else {
+                director.time = Mathf.lerp(director.time, time, this.context.time.deltaTime / this.damping);
+            }
+
+            if (debug && this.context.time.frame % 30 === 0) {
+                console.log(`[ScrollFollow ] Timeline ${director.name}: ${time.toFixed(3)}`, weightsArray.map(w => `[${w.name} ${(w.weight * 100).toFixed(0)}%]`).join(", "));
             }
-            director.time = time;
         }
     }
 
 }
 
+
+
 const weightsArray: OverlapInfo[] = [];
-const markersArray: (ScrollMarkerModel & { element?: HTMLElement | null })[] = [];
+const markersArray: Array<ScrollMarkerModel & {
+    element?: HTMLElement | null,
+    timeline?: ViewTimeline,
+}> = [];
 
 type OverlapInfo = {
+    name: string,
     /** Marker time */
     time: number,
     /** Overlap in pixels */
     weight: number,
+}
+
+
+// type SelectorCache = {
+//     /** The selector used to query the *elements */
+//     selector: string,
+//     elements: Element[] | null,
+//     usedElementCount: number,
+// }
+// const querySelectorResults: Array<SelectorCache> = [];
+
+const needleScrollMarkerIndexCache = new Map<number, Element | null>();
+const needleScrollMarkerNameCache = new Map<string, Element | null>();
+let needsScrollMarkerRefresh = true;
+
+function tryGetElementsForSelector(index: number, name: string, _cycle: number = 0): Element | null {
+
+    if (!needsScrollMarkerRefresh) {
+        if (name?.length) {
+            const element = needleScrollMarkerNameCache.get(name) || null;
+            if (element) return element;
+            // const isNumber = !isNaN(Number(name));
+            // if (!isNumber) {
+            // }
+        }
+        const element = needleScrollMarkerIndexCache.get(index) || null;
+        const value = element?.getAttribute("data-timeline-marker");
+        // if (value?.length) {
+        //     if (cycle === 0) {
+        //         // if the HTML marker we found by index does define a different marker name we try to find the correct HTML element by name
+        //         return tryGetElementsForSelector(index, value, 1);
+        //     }
+        //     if (isDevEnvironment()) console.warn(`ScrollMarker name mismatch: expected "${name}", got "${value}"`);
+        // }
+        return element;
+    }
+    needsScrollMarkerRefresh = false;
+    needleScrollMarkerIndexCache.clear();
+    const markers = document.querySelectorAll(`[data-timeline-marker]`);
+    markers.forEach((m, i) => {
+        needleScrollMarkerIndexCache.set(i, m);
+        const name = m.getAttribute("data-timeline-marker");
+        if (name?.length) needleScrollMarkerNameCache.set(name, m);
+    });
+    needsScrollMarkerRefresh = false;
+    return tryGetElementsForSelector(index, name);
+}
+
+
+// #region ScrollTimeline
+
+function calculateTimelinePositionNormalized(timeline: ViewTimeline) {
+    if (!timeline.source) return 0;
+    const currentTime = timeline.currentTime;
+    const duration = timeline.duration;
+    let durationValue = 1;
+    if (duration.unit === "seconds") {
+        durationValue = duration.value;
+    }
+    else if (duration.unit === "percent") {
+        durationValue = duration.value;
+    }
+    const t01 = currentTime.unit === "seconds" ? (currentTime.value / durationValue) : (currentTime.value / 100);
+    return t01;
+}
+
+
+declare global {
+    interface ViewTimeline {
+        axis: 'block' | 'inline';
+        currentTime: { unit: 'seconds' | 'percent', value: number };
+        duration: { unit: 'seconds' | 'percent', value: number };
+        source: Element | null;
+        startOffset: { unit: 'px', value: number };
+        endOffset: { unit: 'px', value: number };
+    }
 }