@editframe/elements 0.14.0-beta.3 → 0.15.0-beta.1

package/src/elements/EFMedia.ts

@@ -17,6 +17,41 @@ import { FetchMixin } from "./FetchMixin.js";
 
 const log = debug("ef:elements:EFMedia");
 
+const freqWeightsCache = new Map<number, Float32Array>();
+
+class LRUCache<K, V> {
+  private cache = new Map<K, V>();
+  private readonly maxSize: number;
+
+  constructor(maxSize: number) {
+    this.maxSize = maxSize;
+  }
+
+  get(key: K): V | undefined {
+    const value = this.cache.get(key);
+    if (value) {
+      // Refresh position by removing and re-adding
+      this.cache.delete(key);
+      this.cache.set(key, value);
+    }
+    return value;
+  }
+
+  set(key: K, value: V): void {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.maxSize) {
+      // Remove oldest entry (first item in map)
+      const firstKey = this.cache.keys().next().value;
+      this.cache.delete(firstKey);
+    }
+    this.cache.set(key, value);
+  }
+}
+
+// Cache individual frame analyses
+const frequencyDataCache = new LRUCache<string, Uint8Array>(100);
+
 export const deepGetMediaElements = (
   element: Element,
   medias: EFMedia[] = [],
@@ -309,10 +344,13 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         this.endTimeMs < timelineTimeMs
       ) {
         this.style.display = "none";
+        // this.style.zIndex = "";
+        // this.style.opacity = "0";
         return;
       }
       this.style.display = "";
-
+      // this.style.zIndex = "100000";
+      // this.style.opacity = "";
       const animations = this.getAnimations({ subtree: true });
 
       this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
@@ -546,4 +584,128 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         this.trimEndMs,
     };
   }
+
+  @property({ type: Number })
+  fftSize = 512; // Default FFT size
+
+  @property({ type: Number })
+  fftDecay = 8; // Default number of frames to analyze
+
+  private static readonly MIN_DB = -90;
+  private static readonly MAX_DB = -20;
+  private static readonly DECAY_WEIGHT = 0.7;
+
+  // Update FREQ_WEIGHTS to use the instance fftSize instead of a static value
+  get FREQ_WEIGHTS() {
+    if (freqWeightsCache.has(this.fftSize)) {
+      // biome-ignore lint/style/noNonNullAssertion: Will exist due to prior has check
+      return freqWeightsCache.get(this.fftSize)!;
+    }
+
+    const weights = new Float32Array(this.fftSize / 2).map((_, i) => {
+      const frequency = (i * 48000) / this.fftSize;
+      if (frequency < 60) return 0.3;
+      if (frequency < 250) return 0.4;
+      if (frequency < 500) return 0.6;
+      if (frequency < 2000) return 0.8;
+      if (frequency < 4000) return 1.2;
+      if (frequency < 8000) return 1.6;
+      return 2.0;
+    });
+
+    freqWeightsCache.set(this.fftSize, weights);
+    return weights;
+  }
+
+  frequencyDataTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [this.audioBufferTask.status, this.trimAdjustedOwnCurrentTimeMs] as const,
+    task: async () => {
+      await this.audioBufferTask.taskComplete;
+      if (!this.audioBufferTask.value) return null;
+      if (this.trimAdjustedOwnCurrentTimeMs <= 0) return null;
+
+      const currentTimeMs = this.trimAdjustedOwnCurrentTimeMs;
+      const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
+      const audioBuffer = this.audioBufferTask.value.buffer;
+
+      const framesData = await Promise.all(
+        Array.from({ length: this.fftDecay }, async (_, i) => {
+          const frameOffset = i * (1000 / 30);
+          const startTime = Math.max(
+            0,
+            (currentTimeMs - frameOffset - startOffsetMs) / 1000,
+          );
+
+          // Cache key for this specific frame
+          const cacheKey = `${startOffsetMs},${startTime}`;
+
+          // Check cache for this specific frame
+          const cachedFrame = frequencyDataCache.get(cacheKey);
+          if (cachedFrame) {
+            return cachedFrame;
+          }
+
+          const audioContext = new OfflineAudioContext(
+            2,
+            48000 * (1 / 30),
+            48000,
+          );
+          const analyser = audioContext.createAnalyser();
+          analyser.fftSize = this.fftSize;
+          analyser.minDecibels = EFMedia.MIN_DB;
+          analyser.maxDecibels = EFMedia.MAX_DB;
+
+          const audioBufferSource = audioContext.createBufferSource();
+          audioBufferSource.buffer = audioBuffer;
+
+          audioBufferSource.connect(analyser);
+          analyser.connect(audioContext.destination);
+
+          audioBufferSource.start(0, startTime, 1 / 30);
+
+          try {
+            await audioContext.startRendering();
+            const frameData = new Uint8Array(analyser.frequencyBinCount);
+            analyser.getByteFrequencyData(frameData);
+
+            // Cache this frame's analysis
+            frequencyDataCache.set(cacheKey, frameData);
+            return frameData;
+          } finally {
+            audioBufferSource.disconnect();
+            analyser.disconnect();
+          }
+        }),
+      );
+
+      const frameLength = framesData[0]?.length ?? 0;
+
+      // Combine frames with decay
+      const smoothedData = new Uint8Array(frameLength);
+      for (let i = 0; i < frameLength; i++) {
+        let weightedSum = 0;
+        let weightSum = 0;
+
+        framesData.forEach((frame, frameIndex) => {
+          const decayWeight = EFMedia.DECAY_WEIGHT ** frameIndex;
+          // biome-ignore lint/style/noNonNullAssertion: Will exist due to forEach
+          weightedSum += frame[i]! * decayWeight;
+          weightSum += decayWeight;
+        });
+
+        smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+      }
+
+      // Apply frequency weights using instance FREQ_WEIGHTS
+      smoothedData.forEach((value, i) => {
+        // biome-ignore lint/style/noNonNullAssertion: Trusting FREQ_WEIGHTS to be the correct length
+        const freqWeight = this.FREQ_WEIGHTS[i]!;
+        smoothedData[i] = Math.min(255, Math.round(value * freqWeight));
+      });
+
+      return smoothedData;
+    },
+  });
 }

package/src/elements/EFTemporal.ts

@@ -1,5 +1,5 @@
 import { consume, createContext } from "@lit/context";
-import type { LitElement, ReactiveController } from "lit";
+import type { LitElement, PropertyValueMap, ReactiveController } from "lit";
 import { property, state } from "lit/decorators.js";
 import type { EFTimegroup } from "./EFTimegroup.js";
 
@@ -13,6 +13,10 @@ export const timegroupContext = createContext<EFTimegroup>(
 
 export declare class TemporalMixinInterface {
   get hasOwnDuration(): boolean;
+  /**
+   * Whether the element has a duration set as an attribute.
+   */
+  get hasExplicitDuration(): boolean;
 
   /**
    * Used to trim the start of the media.
@@ -461,6 +465,10 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
       return parent as EFTimegroup | undefined;
     }
 
+    get hasExplicitDuration() {
+      return this._durationMs !== undefined;
+    }
+
     get hasOwnDuration() {
       return false;
     }
@@ -613,6 +621,30 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
         }
       },
     });
+
+    protected updated(
+      changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+    ): void {
+      super.updated(changedProperties);
+      if (
+        changedProperties.has("currentTime") ||
+        changedProperties.has("ownCurrentTimeMs")
+      ) {
+        const timelineTimeMs = (this.rootTimegroup ?? this).ownCurrentTimeMs;
+        if (
+          this.startTimeMs >= timelineTimeMs ||
+          this.endTimeMs <= timelineTimeMs
+        ) {
+          this.style.display = "none";
+          // this.style.zIndex = "";
+          // this.style.opacity = "0";
+          return;
+        }
+        this.style.display = "";
+        // this.style.zIndex = "100000";
+        // this.style.opacity = "";
+      }
+    }
   }
 
   Object.defineProperty(TemporalMixinClass.prototype, EF_TEMPORAL, {

package/src/elements/EFTimegroup.ts

@@ -39,7 +39,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
       display: block;
       width: 100%;
       height: 100%;
-      position: relative;
+      position: absolute;
       transform-origin: center center;
     }
   `;
@@ -258,10 +258,13 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
     if (this.startTimeMs > timelineTimeMs || this.endTimeMs < timelineTimeMs) {
       this.style.display = "none";
+      // this.style.zIndex = "";
+      // this.style.opacity = "0";
       return;
     }
     this.style.display = "";
-
+    // this.style.zIndex = "100000";
+    // this.style.opacity = "";
     const animations = this.getAnimations({ subtree: true });
     this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
     this.style.setProperty(