@editframe/elements 0.14.0-beta.3 → 0.15.0-beta.10

package/src/elements/EFMedia.ts

@@ -14,9 +14,44 @@ import { EF_RENDERING } from "../EF_RENDERING.js";
 import { EFSourceMixin } from "./EFSourceMixin.js";
 import { EFTemporal, isEFTemporal } from "./EFTemporal.js";
 import { FetchMixin } from "./FetchMixin.js";
+import { EFTargetable } from "./TargetController.ts";
 
 const log = debug("ef:elements:EFMedia");
 
+const freqWeightsCache = new Map<number, Float32Array>();
+
+class LRUCache<K, V> {
+  private cache = new Map<K, V>();
+  private readonly maxSize: number;
+
+  constructor(maxSize: number) {
+    this.maxSize = maxSize;
+  }
+
+  get(key: K): V | undefined {
+    const value = this.cache.get(key);
+    if (value) {
+      // Refresh position by removing and re-adding
+      this.cache.delete(key);
+      this.cache.set(key, value);
+    }
+    return value;
+  }
+
+  set(key: K, value: V): void {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.maxSize) {
+      // Remove oldest entry (first item in map)
+      const firstKey = this.cache.keys().next().value;
+      if (firstKey) {
+        this.cache.delete(firstKey);
+      }
+    }
+    this.cache.set(key, value);
+  }
+}
+
 export const deepGetMediaElements = (
   element: Element,
   medias: EFMedia[] = [],
@@ -31,9 +66,11 @@ export const deepGetMediaElements = (
   return medias;
 };
 
-export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
-  assetType: "isobmff_files",
-}) {
+export class EFMedia extends EFTargetable(
+  EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
+    assetType: "isobmff_files",
+  }),
+) {
   static styles = [
     css`
       :host {
@@ -284,7 +321,8 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     },
   });
 
-  @state() desiredSeekTimeMs = 0;
+  @state()
+  desiredSeekTimeMs = 0;
 
   protected async executeSeek(seekToMs: number) {
     this.desiredSeekTimeMs = seekToMs;
@@ -294,7 +332,7 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
   ): void {
     if (changedProperties.has("ownCurrentTimeMs")) {
-      this.executeSeek(this.trimAdjustedOwnCurrentTimeMs);
+      this.executeSeek(this.currentSourceTimeMs);
     }
     // TODO: this is copied straight from EFTimegroup.ts
     // and should be refactored to be shared/reduce bad duplication of
@@ -312,7 +350,6 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         return;
       }
       this.style.display = "";
-
       const animations = this.getAnimations({ subtree: true });
 
       this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
@@ -546,4 +583,265 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         this.trimEndMs,
     };
   }
+
+  @property({ type: Number })
+  fftSize = 512; // Default FFT size
+
+  @property({ type: Number })
+  fftDecay = 8; // Default number of frames to analyze
+
+  private static readonly MIN_DB = -90;
+  private static readonly MAX_DB = -20;
+  private static readonly DECAY_WEIGHT = 0.7;
+
+  // Update FREQ_WEIGHTS to use the instance fftSize instead of a static value
+  get FREQ_WEIGHTS() {
+    if (freqWeightsCache.has(this.fftSize)) {
+      // biome-ignore lint/style/noNonNullAssertion: Will exist due to prior has check
+      return freqWeightsCache.get(this.fftSize)!;
+    }
+
+    const weights = new Float32Array(this.fftSize / 2).map((_, i) => {
+      const frequency = (i * 48000) / this.fftSize;
+      if (frequency < 60) return 0.3;
+      if (frequency < 250) return 0.4;
+      if (frequency < 500) return 0.6;
+      if (frequency < 2000) return 0.8;
+      if (frequency < 4000) return 1.2;
+      if (frequency < 8000) return 1.6;
+      return 2.0;
+    });
+
+    freqWeightsCache.set(this.fftSize, weights);
+    return weights;
+  }
+
+  #byteTimeDomainCache = new LRUCache<string, Uint8Array>(100);
+
+  byteTimeDomainTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [
+        this.audioBufferTask.status,
+        this.currentSourceTimeMs,
+        this.fftSize,
+        this.fftDecay,
+      ] as const,
+    task: async () => {
+      await this.audioBufferTask.taskComplete;
+      if (!this.audioBufferTask.value) return null;
+      if (this.currentSourceTimeMs <= 0) return null;
+
+      const currentTimeMs = this.currentSourceTimeMs;
+      const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
+      const audioBuffer = this.audioBufferTask.value.buffer;
+      const smoothedKey = `${this.fftSize}:${this.fftDecay}:${startOffsetMs}:${currentTimeMs}`;
+
+      const cachedSmoothedData = this.#byteTimeDomainCache.get(smoothedKey);
+      if (cachedSmoothedData) {
+        return cachedSmoothedData;
+      }
+
+      const framesData = await Promise.all(
+        Array.from({ length: this.fftDecay }, async (_, i) => {
+          const frameOffset = i * (1000 / 30);
+          const startTime = Math.max(
+            0,
+            (currentTimeMs - frameOffset - startOffsetMs) / 1000,
+          );
+
+          const cacheKey = `${this.fftSize}:${startOffsetMs}:${startTime}`;
+          const cachedFrame = this.#byteTimeDomainCache.get(cacheKey);
+          if (cachedFrame) {
+            return cachedFrame;
+          }
+
+          const audioContext = new OfflineAudioContext(
+            2,
+            48000 * (1 / 30),
+            48000,
+          );
+          const analyser = audioContext.createAnalyser();
+          analyser.fftSize = this.fftSize;
+
+          // Increase gain even more for better signal
+          const gainNode = audioContext.createGain();
+          gainNode.gain.value = 10.0; // Try a higher gain
+
+          // More aggressive settings for the analyzer
+          analyser.smoothingTimeConstant = 0.4;
+          analyser.minDecibels = -90;
+          analyser.maxDecibels = -10;
+
+          const audioBufferSource = audioContext.createBufferSource();
+          audioBufferSource.buffer = audioBuffer;
+
+          // Add a bandpass filter to focus on the most active frequency ranges
+          const filter = audioContext.createBiquadFilter();
+          filter.type = "bandpass";
+          filter.frequency.value = 1000; // Center frequency in Hz
+          filter.Q.value = 0.5; // Width of the band
+
+          audioBufferSource.connect(gainNode);
+          gainNode.connect(filter);
+          filter.connect(analyser);
+          analyser.connect(audioContext.destination);
+
+          audioBufferSource.start(0, startTime, 1 / 30);
+
+          try {
+            await audioContext.startRendering();
+            // Change to time domain data
+            const frameData = new Uint8Array(analyser.fftSize);
+            analyser.getByteTimeDomainData(frameData);
+
+            this.#byteTimeDomainCache.set(cacheKey, frameData);
+            return frameData;
+          } finally {
+            audioBufferSource.disconnect();
+            analyser.disconnect();
+          }
+        }),
+      );
+
+      const frameLength = framesData[0]?.length ?? 0;
+      const smoothedData = new Uint8Array(frameLength);
+
+      // Combine frames with decay
+      for (let i = 0; i < frameLength; i++) {
+        let weightedSum = 0;
+        let weightSum = 0;
+
+        framesData.forEach((frame, frameIndex) => {
+          const decayWeight = EFMedia.DECAY_WEIGHT ** frameIndex;
+          // biome-ignore lint/style/noNonNullAssertion: Will exist due to forEach
+          weightedSum += frame[i]! * decayWeight;
+          weightSum += decayWeight;
+        });
+
+        smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+      }
+
+      // Remove frequency weighting since we're using time domain data
+      // No need to slice the data either since we want the full waveform
+
+      this.#byteTimeDomainCache.set(
+        smoothedKey,
+        smoothedData.slice(0, Math.floor(smoothedData.length * 0.8)),
+      );
+      return smoothedData;
+    },
+  });
+
+  #frequencyDataCache = new LRUCache<string, Uint8Array>(100);
+
+  frequencyDataTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [
+        this.audioBufferTask.status,
+        this.currentSourceTimeMs,
+        this.fftSize, // Add fftSize to dependency array
+        this.fftDecay, // Add fftDecay to dependency array
+      ] as const,
+    task: async () => {
+      await this.audioBufferTask.taskComplete;
+      if (!this.audioBufferTask.value) return null;
+      if (this.currentSourceTimeMs <= 0) return null;
+
+      const currentTimeMs = this.currentSourceTimeMs;
+      const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
+      const audioBuffer = this.audioBufferTask.value.buffer;
+      const smoothedKey = `${this.fftSize}:${this.fftDecay}:${startOffsetMs}:${currentTimeMs}`;
+
+      const cachedSmoothedData = this.#frequencyDataCache.get(smoothedKey);
+      if (cachedSmoothedData) {
+        return cachedSmoothedData;
+      }
+
+      const framesData = await Promise.all(
+        Array.from({ length: this.fftDecay }, async (_, i) => {
+          const frameOffset = i * (1000 / 30);
+          const startTime = Math.max(
+            0,
+            (currentTimeMs - frameOffset - startOffsetMs) / 1000,
+          );
+
+          // Cache key for this specific frame
+          const cacheKey = `${this.fftSize}:${startOffsetMs}:${startTime}`;
+
+          // Check cache for this specific frame
+          const cachedFrame = this.#frequencyDataCache.get(cacheKey);
+          if (cachedFrame) {
+            return cachedFrame;
+          }
+
+          const audioContext = new OfflineAudioContext(
+            2,
+            48000 * (1 / 30),
+            48000,
+          );
+          const analyser = audioContext.createAnalyser();
+          analyser.fftSize = this.fftSize;
+          analyser.minDecibels = EFMedia.MIN_DB;
+          analyser.maxDecibels = EFMedia.MAX_DB;
+
+          const audioBufferSource = audioContext.createBufferSource();
+          audioBufferSource.buffer = audioBuffer;
+
+          audioBufferSource.connect(analyser);
+          analyser.connect(audioContext.destination);
+
+          audioBufferSource.start(0, startTime, 1 / 30);
+
+          try {
+            await audioContext.startRendering();
+            const frameData = new Uint8Array(this.fftSize / 2);
+            analyser.getByteFrequencyData(frameData);
+
+            // Cache this frame's analysis
+            this.#frequencyDataCache.set(cacheKey, frameData);
+            return frameData;
+          } finally {
+            audioBufferSource.disconnect();
+            analyser.disconnect();
+          }
+        }),
+      );
+
+      const frameLength = framesData[0]?.length ?? 0;
+
+      // Combine frames with decay
+      const smoothedData = new Uint8Array(frameLength);
+      for (let i = 0; i < frameLength; i++) {
+        let weightedSum = 0;
+        let weightSum = 0;
+
+        framesData.forEach((frame, frameIndex) => {
+          const decayWeight = EFMedia.DECAY_WEIGHT ** frameIndex;
+          // biome-ignore lint/style/noNonNullAssertion: Will exist due to forEach
+          weightedSum += frame[i]! * decayWeight;
+          weightSum += decayWeight;
+        });
+
+        smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+      }
+
+      // Apply frequency weights using instance FREQ_WEIGHTS
+      smoothedData.forEach((value, i) => {
+        // biome-ignore lint/style/noNonNullAssertion: Will exist due to forEach
+        const freqWeight = this.FREQ_WEIGHTS[i]!;
+        smoothedData[i] = Math.min(255, Math.round(value * freqWeight));
+      });
+
+      // Only return the lower half of the frequency data
+      // The top half is zeroed out, which makes for aesthetically unpleasing waveforms
+      const slicedData = smoothedData.slice(
+        0,
+        Math.floor(smoothedData.length / 2),
+      );
+      this.#frequencyDataCache.set(smoothedKey, slicedData);
+      return slicedData;
+    },
+  });
 }

package/src/elements/EFTemporal.ts

@@ -1,5 +1,5 @@
 import { consume, createContext } from "@lit/context";
-import type { LitElement, ReactiveController } from "lit";
+import type { LitElement, PropertyValueMap, ReactiveController } from "lit";
 import { property, state } from "lit/decorators.js";
 import type { EFTimegroup } from "./EFTimegroup.js";
 
@@ -13,6 +13,10 @@ export const timegroupContext = createContext<EFTimegroup>(
 
 export declare class TemporalMixinInterface {
   get hasOwnDuration(): boolean;
+  /**
+   * Whether the element has a duration set as an attribute.
+   */
+  get hasExplicitDuration(): boolean;
 
   /**
    * Used to trim the start of the media.
@@ -147,18 +151,18 @@ export declare class TemporalMixinInterface {
    * elements.
    *
    * For example, if the media has a `sourcein` value of 10s, when `ownCurrentTimeMs` is 0s,
-   * `trimAdjustedOwnCurrentTimeMs` will be 10s.
+   * `currentSourceTimeMs` will be 10s.
    *
    *            sourcein=10s     sourceout=10s
    * /         /                /
    * |--------|=================|---------|
    *          ^
    *          |_
-   *            trimAdjustedOwnCurrentTimeMs === 10s
+   *            currentSourceTimeMs === 10s
    *          |_
    *            ownCurrentTimeMs === 0s
    */
-  get trimAdjustedOwnCurrentTimeMs(): number;
+  get currentSourceTimeMs(): number;
 
   set duration(value: string);
   get duration(): string;
@@ -461,6 +465,10 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
       return parent as EFTimegroup | undefined;
     }
 
+    get hasExplicitDuration() {
+      return this._durationMs !== undefined;
+    }
+
     get hasOwnDuration() {
       return false;
     }
@@ -559,6 +567,10 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
       return this.startTimeMs + this.durationMs;
     }
 
+    /**
+     * The current time of the element within itself.
+     * Compare with `currentTimeMs` to see the current time with respect to the root timegroup
+     */
     get ownCurrentTimeMs() {
       if (this.rootTimegroup) {
         return Math.min(
@@ -573,7 +585,7 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
      * Used to calculate the internal currentTimeMs of the element. This is useful
      * for mapping to internal media time codes for audio/video elements.
      */
-    get trimAdjustedOwnCurrentTimeMs() {
+    get currentSourceTimeMs() {
       if (this.rootTimegroup) {
         if (this.sourceInMs && this.sourceOutMs) {
           return Math.min(
@@ -613,6 +625,26 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
         }
       },
     });
+
+    protected updated(
+      changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+    ): void {
+      super.updated(changedProperties);
+      if (
+        changedProperties.has("currentTime") ||
+        changedProperties.has("ownCurrentTimeMs")
+      ) {
+        const timelineTimeMs = (this.rootTimegroup ?? this).ownCurrentTimeMs;
+        if (
+          this.startTimeMs > timelineTimeMs ||
+          this.endTimeMs < timelineTimeMs
+        ) {
+          this.style.display = "none";
+          return;
+        }
+        this.style.display = "";
+      }
+    }
   }
 
   Object.defineProperty(TemporalMixinClass.prototype, EF_TEMPORAL, {

package/src/elements/EFTimegroup.ts

@@ -39,7 +39,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
       display: block;
       width: 100%;
       height: 100%;
-      position: relative;
+      position: absolute;
       transform-origin: center center;
     }
   `;
@@ -53,7 +53,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     type: String,
     attribute: "mode",
   })
-  mode: "fixed" | "sequence" | "contain" = "sequence";
+  mode: "fixed" | "sequence" | "contain" = "contain";
 
   @property({
     type: Number,
@@ -67,7 +67,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
 
   #resizeObserver?: ResizeObserver;
 
-  @property({ type: Number })
+  @property({ type: Number, attribute: "currenttime" })
   set currentTime(time: number) {
     this.#currentTime = Math.max(0, Math.min(time, this.durationMs / 1000));
     try {
@@ -227,10 +227,9 @@ export class EFTimegroup extends EFTemporal(LitElement) {
    * in calculations and it was not clear why.
    */
   async waitForMediaDurations() {
+    const mediaElements = deepGetMediaElements(this);
     return await Promise.all(
-      deepGetMediaElements(this).map(
-        (media) => media.initSegmentsLoader.taskComplete,
-      ),
+      mediaElements.map((m) => m.trackFragmentIndexLoader.taskComplete),
     );
   }
 
@@ -261,7 +260,6 @@ export class EFTimegroup extends EFTemporal(LitElement) {
       return;
     }
     this.style.display = "";
-
     const animations = this.getAnimations({ subtree: true });
     this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
     this.style.setProperty(