@needle-tools/engine 4.10.0-beta → 4.10.0-beta.2

package/src/engine-components/web/ScrollFollow.ts

@@ -1,13 +1,14 @@
 import { Box3, Object3D } from "three";
 import { element } from "three/src/nodes/TSL.js";
-import { Context } from "../../engine/engine_context.js";
 
+import { Context } from "../../engine/engine_context.js";
 import { Mathf } from "../../engine/engine_math.js";
 import { serializable } from "../../engine/engine_serialization.js";
-import { getBoundingBox } from "../../engine/engine_three_utils.js";
+import { getBoundingBox, setVisibleInCustomShadowRendering } from "../../engine/engine_three_utils.js";
 import { getParam } from "../../engine/engine_utils.js";
 import { Animation } from "../Animation.js";
 import { Animator } from "../Animator.js";
+import { MarkerTrackHandler } from "../api.js";
 import { AudioSource } from "../AudioSource.js";
 import { Behaviour } from "../Component.js";
 import { EventList } from "../EventList.js";
@@ -150,7 +151,8 @@ export class ScrollFollow extends Behaviour {
             }
         }
 
-        if (this._current_value !== this._appliedValue) {
+        // if (this._current_value !== this._appliedValue) 
+        {
             this._appliedValue = this._current_value;
 
             let defaultPrevented = false;
@@ -242,7 +244,8 @@ export class ScrollFollow extends Behaviour {
 
         if (target instanceof PlayableDirector) {
             this.handleTimelineTarget(target, value);
-            if (!target.isPlaying) target.evaluate();
+            if (target.isPlaying) target.pause();
+            target.evaluate();
         }
         else if (target instanceof Animator) {
             target.setFloat("scroll", value);
@@ -292,18 +295,35 @@ export class ScrollFollow extends Behaviour {
         let scrollRegionEnd = 0;
         markersArray.length = 0;
 
-        for (const marker of director.foreachMarker<ScrollMarkerModel & { element?: HTMLElement | null, needsUpdate?: boolean }>("ScrollMarker")) {
+        // querySelectorResults.length = 0;
+        let markerIndex = 0;
+
+        // https://scroll-driven-animations.style/tools/view-timeline/ranges
+        for (const marker of director.foreachMarker<ScrollMarkerModel & { element?: HTMLElement | null, needsUpdate?: boolean, timeline?: ViewTimeline }>("ScrollMarker")) {
+
+            const index = markerIndex++;
 
             // Get marker elements from DOM
-            if (marker.selector?.length && (marker.element === undefined || marker.needsUpdate === true || /** element is not in DOM anymore? */ (!marker.element?.parentNode))) {
+            if ((marker.element === undefined || marker.needsUpdate === true || /** element is not in DOM anymore? */ (!marker.element?.parentNode))) {
                 marker.needsUpdate = false;
                 try {
-                    marker.element = document.querySelector<HTMLElement>(marker.selector) || null;
-                    if (debug) console.debug("ScrollMarker found on page", marker.element, marker.selector);
+                    marker.element = tryGetElementsForSelector(index, marker.name) as HTMLElement | null;
+                    if (debug) console.debug("ScrollMarker found on page", marker.element, marker.name);
+                    if (!marker.element) {
+                        marker.timeline = undefined;
+                        continue;
+                    }
+                    else {
+                        /** @ts-ignore */
+                        marker.timeline = new ViewTimeline({
+                            subject: marker.element,
+                            axis: 'block', // https://drafts.csswg.org/scroll-animations/#scroll-notation
+                        });
+                    }
                 }
                 catch (error) {
                     marker.element = null;
-                    console.error("ScrollMarker selector is not valid: " + marker.selector + "\n", error);
+                    console.error("ScrollMarker selector is not valid: " + marker.name + "\n", error);
                 }
             }
 
@@ -330,41 +350,74 @@ export class ScrollFollow extends Behaviour {
 
         weightsArray.length = 0;
         let sum = 0;
+        const oneFrameTime = 1 / 60;
 
+        // We keep a separate count here in case there are some markers that could not be resolved so point to *invalid* elements - the timeline should fallback to 0-1 scroll behaviour then
         let markerCount = 0;
-        for (const marker of markersArray) {
-
+        for (let i = 0; i < markersArray.length; i++) {
+            const marker = markersArray[i];
             if (!marker.element) continue;
+            const nextMarker = markersArray[i + 1];
 
-            const top = marker.element.offsetTop;
-            const height = marker.element.offsetHeight;
-            const bottom = top + height;
-            let overlap = 0;
-
-            // TODO: if we have two marker sections where no HTML overlaps (vor example because some large section is between them) we probably want to still virtually interpolate between them slowly in that region
-
-            if (bottom < currentTop) {
-                // marker is above scroll region
-                overlap = 0;
-            }
-            else if (top > currentBottom) {
-                // marker is below scroll region
-                overlap = 0;
-            }
-            else {
-                // calculate overlap in pixels
-                const overlapTop = Math.max(top, currentTop);
-                const overlapBottom = Math.min(bottom, currentBottom);
-                overlap = Math.max(0, overlapBottom - overlapTop);
-                // console.log(marker.element.className, overlap)
-            }
+            const nextTime = nextMarker
+                ? (nextMarker.time - oneFrameTime)
+                : duration;
 
             markerCount += 1;
 
-            if (overlap > 0) {
-                weightsArray.push({ time: marker.time, weight: overlap });
-                sum += overlap;
+            const timeline = marker.timeline;
+            if (timeline) {
+                const time01 = calculateTimelinePositionNormalized(timeline);
+                // remap 0-1 to 0 - 1 - 0 (full weight at center)
+                const weight = 1 - Math.abs(time01 - 0.5) * 2;
+                if (time01 > 0 && time01 <= 1) {
+                    const lerpTime = marker.time + (nextTime - marker.time) * time01;
+                    weightsArray.push({ time: lerpTime, weight: weight });
+                    sum += weight;
+                }
+                // Before the first marker is reached
+                else if (i === 0 && time01 <= 0) {
+                    weightsArray.push({ time: 0, weight: 1 });
+                    sum += 1;
+                }
+                // After the last marker is reached
+                else if (i === markersArray.length - 1 && time01 >= 1) {
+                    weightsArray.push({ time: duration, weight: 1 });
+                    sum += 1;
+                }
             }
+            continue;
+            // if(this.context.time.frame % 10 === 0) console.log(marker.element?.className, timeline, calculateTimelinePositionNormalized(timeline!));
+
+            // const top = marker.element.offsetTop - this._scrollContainerHeight;
+            // const height = marker.element.offsetHeight + this._scrollContainerHeight;
+            // const bottom = top + height;
+            // let overlap = 0;
+
+            // // TODO: if we have two marker sections where no HTML overlaps (vor example because some large section is between them) we probably want to still virtually interpolate between them slowly in that region
+
+            // if (bottom < currentTop) {
+            //     // marker is above scroll region
+            //     overlap = 0;
+            // }
+            // else if (top > currentBottom) {
+            //     // marker is below scroll region
+            //     overlap = 0;
+            // }
+            // else {
+            //     // calculate overlap in pixels
+            //     const overlapTop = Math.max(top, currentTop);
+            //     const overlapBottom = Math.min(bottom, currentBottom);
+            //     const height = Math.max(1, currentBottom - currentTop);
+            //     overlap = Math.max(0, overlapBottom - overlapTop);
+            // }
+
+            // // if(this.context.time.frame % 20 === 0) console.log(overlap)
+
+            // if (overlap > 0) {
+            //     weightsArray.push({ time: marker.time, weight: overlap });
+            //     sum += overlap;
+            // }
         }
 
         if (weightsArray.length <= 0 && markerCount <= 0) {
@@ -372,25 +425,118 @@ export class ScrollFollow extends Behaviour {
         }
         else if (weightsArray.length > 0) {
             // normalize and calculate weighted time
-            let time = weightsArray[0].time;
-            for (const o of weightsArray) {
-                const weight = o.weight / Math.max(0.00001, sum);
-                // lerp time based on weight
-                const diff = Math.abs(o.time - time);
-                time += diff * weight;
+            let time = weightsArray[0].time; // fallback to first time
+            if (weightsArray.length > 1) {
+                for (const entry of weightsArray) {
+                    const weight = entry.weight / Math.max(0.00001, sum);
+                    // console.log(weight.toFixed(2))
+                    // lerp time based on weight
+                    const diff = Math.abs(entry.time - time);
+                    time += diff * weight;
+                }
+            }
+            if (debug && this.context.time.frame % 20 === 0) console.log(time.toFixed(3), [...weightsArray])
+            if (this.damping <= 0) {
+                director.time = time;
+            }
+            else {
+                director.time = Mathf.lerp(director.time, time, this.context.time.deltaTime / this.damping);
             }
-            director.time = time;
         }
     }
 
 }
 
+
+
 const weightsArray: OverlapInfo[] = [];
-const markersArray: (ScrollMarkerModel & { element?: HTMLElement | null })[] = [];
+const markersArray: (ScrollMarkerModel & { element?: HTMLElement | null, timeline?: ViewTimeline })[] = [];
 
 type OverlapInfo = {
     /** Marker time */
     time: number,
     /** Overlap in pixels */
     weight: number,
+}
+
+
+// type SelectorCache = {
+//     /** The selector used to query the *elements */
+//     selector: string,
+//     elements: Element[] | null,
+//     usedElementCount: number,
+// }
+// const querySelectorResults: Array<SelectorCache> = [];
+
+const needleScrollMarkerCacheKey = "data-timeline-marker";
+const needleScrollMarkerIndexCache = new Map<number, Element | null>();
+const needleScrollMarkerNameCache = new Map<string, Element | null>();
+let needsScrollMarkerRefresh = true;
+
+function tryGetElementsForSelector(index: number, name: string): Element | null {
+
+    if (!needsScrollMarkerRefresh) {
+        let element = name?.length ? needleScrollMarkerNameCache.get(name) : null;
+        if (element) return element;
+        element = needleScrollMarkerIndexCache.get(index) || null;
+        return element;
+    }
+    needsScrollMarkerRefresh = false;
+    needleScrollMarkerIndexCache.clear();
+    const markers = document.querySelectorAll(`[data-timeline-marker]`);
+    markers.forEach((m, i) => {
+        needleScrollMarkerIndexCache.set(i, m);
+        const name = m.getAttribute("data-timeline-marker");
+        if (name?.length) needleScrollMarkerNameCache.set(name, m);
+    });
+    const element = needleScrollMarkerIndexCache.get(index) || null;
+    return element;
+
+
+    /* e.g. 
+    <div class="section behind start" data-needle-scroll-marker>
+    */
+    // console.log(index, element)
+    if (element) return element;
+
+    // for (const entry of querySelectorResults) {
+    //     if (entry.selector === selector) {
+    //         const index = entry.usedElementCount++;
+    //         return entry.elements && index < entry.elements.length ? entry.elements[index] : null;
+    //     }
+    // }
+    // const elements = document.querySelectorAll(selector);
+    // querySelectorResults.push({ selector, elements: Array.from(elements), usedElementCount: 1 });
+    // if (elements.length > 0) return elements[0];
+    return null;
+}
+
+
+// #region ScrollTimeline
+
+function calculateTimelinePositionNormalized(timeline: ViewTimeline) {
+    if (!timeline.source) return 0;
+    const currentTime = timeline.currentTime;
+    const duration = timeline.duration;
+    let durationValue = 1;
+    if (duration.unit === "seconds") {
+        durationValue = duration.value;
+    }
+    else if (duration.unit === "percent") {
+        durationValue = duration.value;
+    }
+    const t01 = currentTime.unit === "seconds" ? (currentTime.value / durationValue) : (currentTime.value / 100);
+    return t01;
+}
+
+
+declare global {
+    interface ViewTimeline {
+        axis: 'block' | 'inline';
+        currentTime: { unit: 'seconds' | 'percent', value: number };
+        duration: { unit: 'seconds' | 'percent', value: number };
+        source: Element | null;
+        startOffset: { unit: 'px', value: number };
+        endOffset: { unit: 'px', value: number };
+    }
 }

package/src/engine-components/web/ViewBox.ts

@@ -0,0 +1,202 @@
+import { Camera, PerspectiveCamera, Vector2, Vector3 } from "three";
+
+import { isDevEnvironment } from "../../engine/debug/debug.js";
+import { Gizmos } from "../../engine/engine_gizmos.js";
+import { serializable } from "../../engine/engine_serialization_decorator.js";
+import { getTempVector } from "../../engine/engine_three_utils.js";
+import { registerType } from "../../engine/engine_typestore.js";
+import { getParam } from "../../engine/engine_utils.js";
+import { Behaviour } from "../Component.js";
+
+const debugParam = getParam("debugviewbox");
+
+@registerType
+export class ViewBox extends Behaviour {
+
+    static instances: ViewBox[] = [];
+
+    @serializable()
+    referenceFieldOfView: number = 60;
+
+    @serializable()
+    debug: boolean = false;
+
+    awake() {
+        // this.referenceFieldOfView = (this.context.mainCamera as PerspectiveCamera)?.fov || 60;
+        // setInterval(()=>{
+        //     this.enabled = !this.enabled
+        // }, 1000)
+    }
+    onEnable(): void {
+        if (debugParam || this.debug || isDevEnvironment()) console.debug("[ViewBox] Using camera fov:", this.referenceFieldOfView);
+        ViewBox.instances.push(this);
+    }
+
+    onDisable(): void {
+        const idx = ViewBox.instances.indexOf(this);
+        if (idx !== -1) ViewBox.instances.splice(idx, 1);
+    }
+
+    onBeforeRender() {
+        if (this.context.isInXR) return;
+        const isActive = ViewBox.instances[ViewBox.instances.length - 1] === this;
+        if (!isActive) {
+            if (debugParam || this.debug) Gizmos.DrawWireBox(this.gameObject.worldPosition, this.gameObject.worldScale, 0x333333);
+            return;
+        }
+        if (debugParam || this.debug) Gizmos.DrawWireBox(this.gameObject.worldPosition, this.gameObject.worldScale, 0xdddd00);
+
+        // calculate box size to fit the camera frustrum size at the current position (just scale)
+        const camera = this.context.mainCamera;
+        if (!camera) return;
+        if (!(camera instanceof PerspectiveCamera)) {
+            // TODO: support orthographic camera
+            return;
+        }
+
+        if (this.referenceFieldOfView === undefined || this.referenceFieldOfView <= 0) {
+            if (debugParam || this.debug) console.warn("[ViewBox] No valid referenceFieldOfView set, cannot adjust box size:", this.referenceFieldOfView);
+            return;
+        }
+
+        const domWidth = this.context.domWidth;
+        const domHeight = this.context.domHeight;
+
+        let rectPosX = 0;
+        let rectPosY = 0;
+        let rectWidth = domWidth;
+        let rectHeight = domHeight;
+        let diffWidth = 1;
+        let diffHeight = 1;
+        // use focus rect if available
+        const focusRectSize = this.context.focusRectSize;
+        if (focusRectSize) {
+            // console.log(focusRectSize)
+            rectPosX = focusRectSize.x;
+            rectPosY = focusRectSize.y;
+            rectWidth = focusRectSize.width;
+            rectHeight = focusRectSize.height;
+            diffWidth = domWidth / rectWidth;
+            diffHeight = domHeight / rectHeight;
+        }
+
+        // const view = camera.view;
+        const view = camera.view;
+        const zoom = camera.zoom;
+        const aspect = camera.aspect;
+        const fov = camera.fov;
+        camera.view = null;
+        camera.zoom = 1;
+        // camera.aspect = rectWidth / rectHeight;
+        camera.fov = this.referenceFieldOfView;
+        camera.updateProjectionMatrix();
+
+        const boxPosition = this.gameObject.worldPosition;
+        const boxScale = this.gameObject.worldScale;
+
+
+
+        // const fov = this.referenceFieldOfView
+        const distance = camera.worldPosition.distanceTo(boxPosition);
+        const vFOV = this.referenceFieldOfView * Math.PI / 180; // convert vertical fov to radians
+        const height = 2 * Math.tan(vFOV / 2) * distance; // visible height
+        const width = height * camera.aspect; // visible width
+
+        const projectedBox = this.projectBoxIntoCamera(boxPosition, boxScale, camera, height * .5);
+        const boxWidth = (projectedBox.maxX - projectedBox.minX);
+        const boxHeight = (projectedBox.maxY - projectedBox.minY);
+
+        // TODO: take the rect size different into account
+        const scale = this.fit(
+            boxWidth * camera.aspect,
+            boxHeight,
+            width / diffWidth,
+            height / diffHeight
+        );
+        const vec = getTempVector(boxPosition);
+        vec.project(camera);
+        this.context.focusRectSettings.offsetX = vec.x;
+        this.context.focusRectSettings.offsetY = vec.y;
+        this.context.focusRectSettings.zoom = scale;
+        // if we don't have a focus rect yet, set it to the dom element
+        if (!this.context.focusRect) this.context.setCameraFocusRect(this.context.domElement);
+
+        // Reset values
+        camera.view = view;
+        camera.zoom = zoom;
+        camera.aspect = aspect;
+        camera.fov = fov;
+        // camera.updateProjectionMatrix();
+
+
+        // BACKLOG: some code for box scale of an object (different component)
+        // this.gameObject.worldScale = getTempVector(width, height, worldscale.z);
+        // this.gameObject.scale.multiplyScalar(.98)
+        // const minscale = Math.min(width, height);
+        // console.log(width, height);
+        // this.gameObject.worldScale = getTempVector(scale, scale, scale);
+    }
+
+
+    /**
+     * Cover fit
+     */
+    private fit(width1: number, height1: number, width2: number, height2: number) {
+        const scaleX = width2 / width1;
+        const scaleY = height2 / height1;
+        return Math.min(scaleX, scaleY);
+    }
+
+
+
+    private projectBoxIntoCamera(position: Vector3, scale: Vector3, camera: Camera, diff: number) {
+
+        const factor = .5 * diff;
+
+        const corners = [
+            getTempVector(-scale.x * factor, -scale.y * factor, -scale.z * factor),
+            getTempVector(scale.x * factor, -scale.y * factor, -scale.z * factor),
+            getTempVector(-scale.x * factor, scale.y * factor, -scale.z * factor),
+            getTempVector(scale.x * factor, scale.y * factor, -scale.z * factor),
+            getTempVector(-scale.x * factor, -scale.y * factor, scale.z * factor),
+            getTempVector(scale.x * factor, -scale.y * factor, scale.z * factor),
+            getTempVector(-scale.x * factor, scale.y * factor, scale.z * factor),
+            getTempVector(scale.x * factor, scale.y * factor, scale.z * factor),
+        ];
+        let minX = Number.POSITIVE_INFINITY;
+        let maxX = Number.NEGATIVE_INFINITY;
+        let minY = Number.POSITIVE_INFINITY;
+        let maxY = Number.NEGATIVE_INFINITY;
+        for (let i = 0; i < corners.length; i++) {
+            const c = corners[i];
+            c.add(position);
+            c.project(camera);
+            if (c.x < minX) minX = c.x;
+            if (c.x > maxX) maxX = c.x;
+            if (c.y < minY) minY = c.y;
+            if (c.y > maxY) maxY = c.y;
+        }
+
+        // if(!this._projectedBoxElement) {
+        //     this._projectedBoxElement = document.createElement("div");
+        // this.context.domElement.appendChild(this._projectedBoxElement);
+        // }
+        // this._projectedBoxElement.style.position = "fixed";
+        // this._projectedBoxElement.style.outline = "10px solid red";
+        // this._projectedBoxElement.style.left = ((minX * .5 + .5) * this.context.domWidth) + "px";
+        // this._projectedBoxElement.style.top = ((-maxY * .5 + .5) * this.context.domHeight) + "px";
+        // this._projectedBoxElement.style.width = ((maxX - minX) * .5 * this.context.domWidth) + "px";
+        // this._projectedBoxElement.style.height = ((maxY - minY) * .5 * this.context.domHeight) + "px";
+        // this._projectedBoxElement.style.pointerEvents = "none";
+        // this._projectedBoxElement.style.zIndex = "1000";
+
+
+        return { minX, maxX, minY, maxY };
+
+    }
+    private _projectedBoxElement: HTMLElement | null = null;
+
+
+
+
+}

package/src/engine-components/web/index.ts

@@ -1,4 +1,5 @@
 export * from "./Clickthrough.js";
 export * from "./CursorFollow.js";
 export * from "./HoverAnimation.js";
-export * from "./ScrollFollow.js";
+export * from "./ScrollFollow.js";
+export * from "./ViewBox.js";