@editframe/elements 0.15.0-beta.8 → 0.16.0-beta.0

package/src/elements/EFMedia.ts

@@ -10,11 +10,11 @@ import type { TrackFragmentIndex, TrackSegment } from "@editframe/assets";
 import { VideoAsset } from "@editframe/assets/EncodedAsset.js";
 import { MP4File } from "@editframe/assets/MP4File.js";
 import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
-import { EF_RENDERING } from "../EF_RENDERING.js";
 import { EFSourceMixin } from "./EFSourceMixin.js";
-import { EFTemporal, isEFTemporal } from "./EFTemporal.js";
+import { EFTemporal } from "./EFTemporal.js";
 import { FetchMixin } from "./FetchMixin.js";
 import { EFTargetable } from "./TargetController.ts";
+import { updateAnimations } from "./updateAnimations.ts";
 
 const log = debug("ef:elements:EFMedia");
 
@@ -102,9 +102,6 @@ export class EFMedia extends EFTargetable(
 
   fragmentIndexPath() {
     if (this.assetId) {
-      if (EF_RENDERING()) {
-        return `editframe://api/v1/isobmff_files/${this.assetId}/index`;
-      }
       return `${this.apiHost}/api/v1/isobmff_files/${this.assetId}/index`;
     }
     return `/@ef-track-fragment-index/${this.src ?? ""}`;
@@ -112,9 +109,6 @@ export class EFMedia extends EFTargetable(
 
   fragmentTrackPath(trackId: string) {
     if (this.assetId) {
-      if (EF_RENDERING()) {
-        return `editframe://api/v1/isobmff_tracks/${this.assetId}/${trackId}`;
-      }
       return `${this.apiHost}/api/v1/isobmff_tracks/${this.assetId}/${trackId}`;
     }
     // trackId is only specified as a query  in the @ef-track url shape
@@ -126,12 +120,21 @@ export class EFMedia extends EFTargetable(
   public trackFragmentIndexLoader = new Task(this, {
     args: () => [this.fragmentIndexPath(), this.fetch] as const,
     task: async ([fragmentIndexPath, fetch], { signal }) => {
-      const response = await fetch(fragmentIndexPath, { signal });
-      return (await response.json()) as Record<number, TrackFragmentIndex>;
+      try {
+        const response = await fetch(fragmentIndexPath, { signal });
+
+        return (await response.json()) as Record<number, TrackFragmentIndex>;
+      } catch (error) {
+        log("Failed to load track fragment index", error);
+        return undefined;
+      }
     },
     onComplete: () => {
+      this.requestUpdate("intrinsicDurationMs");
       this.requestUpdate("ownCurrentTimeMs");
+      console.log("Requesting update for durationMs", this, this.rootTimegroup);
       this.rootTimegroup?.requestUpdate("ownCurrentTimeMs");
+      this.rootTimegroup?.requestUpdate("durationMs");
     },
   });
 
@@ -341,75 +344,7 @@ export class EFMedia extends EFTargetable(
       changedProperties.has("currentTime") ||
       changedProperties.has("ownCurrentTimeMs")
     ) {
-      const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
-      if (
-        this.startTimeMs > timelineTimeMs ||
-        this.endTimeMs < timelineTimeMs
-      ) {
-        this.style.display = "none";
-        return;
-      }
-      this.style.display = "";
-      const animations = this.getAnimations({ subtree: true });
-
-      this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
-      this.style.setProperty(
-        "--ef-transition-duration",
-        `${this.parentTimegroup?.overlapMs ?? 0}ms`,
-      );
-      this.style.setProperty(
-        "--ef-transition-out-start",
-        `${this.durationMs - (this.parentTimegroup?.overlapMs ?? 0)}ms`,
-      );
-
-      for (const animation of animations) {
-        if (animation.playState === "running") {
-          animation.pause();
-        }
-        const effect = animation.effect;
-        if (!(effect && effect instanceof KeyframeEffect)) {
-          return;
-        }
-        const target = effect.target;
-        // TODO: better generalize work avoidance for temporal elements
-        if (!target) {
-          return;
-        }
-        if (target.closest("ef-video, ef-audio") !== this) {
-          return;
-        }
-
-        // Important to avoid going to the end of the animation
-        // or it will reset awkwardly.
-        if (isEFTemporal(target)) {
-          const timing = effect.getTiming();
-          const duration = Number(timing.duration) ?? 0;
-          const delay = Number(timing.delay);
-          const newTime = Math.floor(
-            Math.min(target.ownCurrentTimeMs, duration - 1 + delay),
-          );
-          if (Number.isNaN(newTime)) {
-            return;
-          }
-          animation.currentTime = newTime;
-        } else if (target) {
-          const nearestTimegroup = target.closest("ef-timegroup");
-          if (!nearestTimegroup) {
-            return;
-          }
-          const timing = effect.getTiming();
-          const duration = Number(timing.duration) ?? 0;
-          const delay = Number(timing.delay);
-          const newTime = Math.floor(
-            Math.min(nearestTimegroup.ownCurrentTimeMs, duration - 1 + delay),
-          );
-
-          if (Number.isNaN(newTime)) {
-            return;
-          }
-          animation.currentTime = newTime;
-        }
-      }
+      updateAnimations(this);
     }
   }
 
@@ -417,7 +352,7 @@ export class EFMedia extends EFTargetable(
     return true;
   }
 
-  get durationMs() {
+  get intrinsicDurationMs() {
     if (!this.trackFragmentIndexLoader.value) {
       return 0;
     }
@@ -430,39 +365,7 @@ export class EFMedia extends EFTargetable(
     if (durations.length === 0) {
       return 0;
     }
-    if (
-      this.sourceInMs &&
-      this.sourceOutMs &&
-      this.sourceOutMs > this.sourceInMs
-    ) {
-      return Math.max(this.sourceOutMs - this.sourceInMs);
-    }
-    if (this.sourceInMs) {
-      return (
-        Math.max(...durations) -
-        this.trimStartMs -
-        this.trimEndMs -
-        this.sourceInMs
-      );
-    }
-    if (this.sourceOutMs) {
-      return (
-        Math.max(...durations) -
-        this.trimStartMs -
-        this.trimEndMs -
-        this.sourceOutMs
-      );
-    }
-    if (this.sourceInMs && this.sourceOutMs) {
-      return (
-        Math.max(...durations) -
-        this.trimStartMs -
-        this.trimEndMs -
-        this.sourceOutMs -
-        this.sourceInMs
-      );
-    }
-    return Math.max(...durations) - this.trimStartMs - this.trimEndMs;
+    return Math.max(...durations);
   }
 
   #audioContext = new OfflineAudioContext(2, 48000 / 30, 48000);
@@ -500,13 +403,15 @@ export class EFMedia extends EFTargetable(
   async fetchAudioSpanningTime(fromMs: number, toMs: number) {
     // Adjust range for track's own time
     if (this.sourceInMs) {
-      fromMs -= this.startTimeMs - this.trimStartMs - this.sourceInMs;
+      fromMs -=
+        this.startTimeMs - (this.trimStartMs ?? 0) - (this.sourceInMs ?? 0);
     }
     if (this.sourceOutMs) {
-      toMs -= this.startTimeMs - this.trimStartMs - this.sourceOutMs;
+      toMs -=
+        this.startTimeMs - (this.trimStartMs ?? 0) - (this.sourceOutMs ?? 0);
     }
-    fromMs -= this.startTimeMs - this.trimStartMs;
-    toMs -= this.startTimeMs - this.trimStartMs;
+    fromMs -= this.startTimeMs - (this.trimStartMs ?? 0);
+    toMs -= this.startTimeMs - (this.trimStartMs ?? 0);
 
     await this.trackFragmentIndexLoader.taskComplete;
     const audioTrackId = this.defaultAudioTrackId;
@@ -576,22 +481,51 @@ export class EFMedia extends EFTargetable(
       blob: audioBlob,
       startMs:
         (firstFragment.dts / audioTrackIndex.timescale) * 1000 -
-        this.trimStartMs,
+        (this.trimStartMs ?? 0),
       endMs:
         (lastFragment.dts / audioTrackIndex.timescale) * 1000 +
         (lastFragment.duration / audioTrackIndex.timescale) * 1000 -
-        this.trimEndMs,
+        (this.trimEndMs ?? 0),
     };
   }
 
-  @property({ type: Number })
-  fftSize = 512; // Default FFT size
+  set fftSize(value: number) {
+    const oldValue = this.fftSize;
+    this.setAttribute("fft-size", String(value));
+    this.requestUpdate("fft-size", oldValue);
+  }
 
-  @property({ type: Number })
-  fftDecay = 8; // Default number of frames to analyze
+  set fftDecay(value: number) {
+    const oldValue = this.fftDecay;
+    this.setAttribute("fft-decay", String(value));
+    this.requestUpdate("fft-decay", oldValue);
+  }
+
+  get fftSize() {
+    return Number.parseInt(this.getAttribute("fft-size") ?? "128", 10);
+  }
+
+  get fftDecay() {
+    return Number.parseInt(this.getAttribute("fft-decay") ?? "8", 10);
+  }
+
+  set interpolateFrequencies(value: boolean) {
+    const oldValue = this.interpolateFrequencies;
+    this.setAttribute("interpolate-frequencies", String(value));
+    this.requestUpdate("interpolate-frequencies", oldValue);
+  }
+
+  get interpolateFrequencies() {
+    return this.getAttribute("interpolate-frequencies") !== "false";
+  }
+
+  get shouldInterpolateFrequencies() {
+    if (this.hasAttribute("interpolate-frequencies")) {
+      return this.getAttribute("interpolate-frequencies") !== "false";
+    }
+    return false;
+  }
 
-  private static readonly MIN_DB = -90;
-  private static readonly MAX_DB = -20;
   private static readonly DECAY_WEIGHT = 0.7;
 
   // Update FREQ_WEIGHTS to use the instance fftSize instead of a static value
@@ -616,6 +550,132 @@ export class EFMedia extends EFTargetable(
     return weights;
   }
 
+  #byteTimeDomainCache = new LRUCache<string, Uint8Array>(100);
+
+  byteTimeDomainTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [
+        this.audioBufferTask.status,
+        this.currentSourceTimeMs,
+        this.fftSize,
+        this.fftDecay,
+        this.fftGain,
+        this.shouldInterpolateFrequencies,
+      ] as const,
+    task: async () => {
+      await this.audioBufferTask.taskComplete;
+      if (!this.audioBufferTask.value) return null;
+      if (this.currentSourceTimeMs <= 0) return null;
+
+      const currentTimeMs = this.currentSourceTimeMs;
+      const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
+      const audioBuffer = this.audioBufferTask.value.buffer;
+
+      const smoothedKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftDecay}:${this.fftGain}:${startOffsetMs}:${currentTimeMs}`;
+      const cachedData = this.#byteTimeDomainCache.get(smoothedKey);
+      if (cachedData) return cachedData;
+
+      // Process multiple frames with decay, similar to the reference code
+      const framesData = await Promise.all(
+        Array.from({ length: this.fftDecay }, async (_, frameIndex) => {
+          const frameOffset = frameIndex * (1000 / 30);
+          const startTime = Math.max(
+            0,
+            (currentTimeMs - frameOffset - startOffsetMs) / 1000,
+          );
+
+          const cacheKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftGain}:${startOffsetMs}:${startTime}`;
+          const cachedFrame = this.#byteTimeDomainCache.get(cacheKey);
+          if (cachedFrame) return cachedFrame;
+
+          const audioContext = new OfflineAudioContext(
+            2,
+            48000 * (1 / 30),
+            48000,
+          );
+
+          const source = audioContext.createBufferSource();
+          source.buffer = audioBuffer;
+
+          // Create analyzer for PCM data
+          const analyser = audioContext.createAnalyser();
+          analyser.fftSize = this.fftSize; // Ensure enough samples
+          analyser.minDecibels = -90;
+          analyser.maxDecibels = -20;
+
+          const gainNode = audioContext.createGain();
+          gainNode.gain.value = this.fftGain; // Amplify the signal
+
+          source.connect(gainNode);
+          gainNode.connect(analyser);
+          analyser.connect(audioContext.destination);
+
+          source.start(0, startTime, 1 / 30);
+
+          const dataLength = analyser.fftSize / 2;
+          try {
+            await audioContext.startRendering();
+            const frameData = new Uint8Array(dataLength);
+            analyser.getByteTimeDomainData(frameData);
+
+            // const points = frameData;
+            // Calculate RMS and midpoint values
+            const points = new Uint8Array(dataLength);
+            for (let i = 0; i < dataLength; i++) {
+              const pointSamples = frameData.slice(
+                i * (frameData.length / dataLength),
+                (i + 1) * (frameData.length / dataLength),
+              );
+
+              // Calculate RMS while preserving sign
+              const rms = Math.sqrt(
+                pointSamples.reduce((sum, sample) => {
+                  const normalized = (sample - 128) / 128;
+                  return sum + normalized * normalized;
+                }, 0) / pointSamples.length,
+              );
+
+              // Get average sign of the samples to determine direction
+              const avgSign = Math.sign(
+                pointSamples.reduce((sum, sample) => sum + (sample - 128), 0),
+              );
+
+              // Convert RMS back to byte range, preserving direction
+              points[i] = Math.min(255, Math.round(128 + avgSign * rms * 128));
+            }
+
+            this.#byteTimeDomainCache.set(cacheKey, points);
+            return points;
+          } finally {
+            source.disconnect();
+            analyser.disconnect();
+          }
+        }),
+      );
+
+      // Combine frames with decay weighting
+      const frameLength = framesData[0]?.length ?? 0;
+      const smoothedData = new Uint8Array(frameLength);
+
+      for (let i = 0; i < frameLength; i++) {
+        let weightedSum = 0;
+        let weightSum = 0;
+
+        framesData.forEach((frame, frameIndex) => {
+          const decayWeight = EFMedia.DECAY_WEIGHT ** frameIndex;
+          weightedSum += (frame[i] ?? 0) * decayWeight;
+          weightSum += decayWeight;
+        });
+
+        smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+      }
+
+      this.#byteTimeDomainCache.set(smoothedKey, smoothedData);
+      return smoothedData;
+    },
+  });
+
   #frequencyDataCache = new LRUCache<string, Uint8Array>(100);
 
   frequencyDataTask = new Task(this, {
@@ -624,8 +684,10 @@ export class EFMedia extends EFTargetable(
       [
         this.audioBufferTask.status,
         this.currentSourceTimeMs,
-        this.fftSize, // Add fftSize to dependency array
-        this.fftDecay, // Add fftDecay to dependency array
+        this.fftSize,
+        this.fftDecay,
+        this.fftGain,
+        this.shouldInterpolateFrequencies,
       ] as const,
     task: async () => {
       await this.audioBufferTask.taskComplete;
@@ -635,7 +697,7 @@ export class EFMedia extends EFTargetable(
       const currentTimeMs = this.currentSourceTimeMs;
       const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
       const audioBuffer = this.audioBufferTask.value.buffer;
-      const smoothedKey = `${this.fftSize}:${this.fftDecay}:${startOffsetMs}:${currentTimeMs}`;
+      const smoothedKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftDecay}:${this.fftGain}:${startOffsetMs}:${currentTimeMs}`;
 
       const cachedSmoothedData = this.#frequencyDataCache.get(smoothedKey);
       if (cachedSmoothedData) {
@@ -651,7 +713,7 @@ export class EFMedia extends EFTargetable(
           );
 
           // Cache key for this specific frame
-          const cacheKey = `${this.fftSize}:${startOffsetMs}:${startTime}`;
+          const cacheKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftGain}:${startOffsetMs}:${startTime}`;
 
           // Check cache for this specific frame
           const cachedFrame = this.#frequencyDataCache.get(cacheKey);
@@ -666,13 +728,23 @@ export class EFMedia extends EFTargetable(
           );
           const analyser = audioContext.createAnalyser();
           analyser.fftSize = this.fftSize;
-          analyser.minDecibels = EFMedia.MIN_DB;
-          analyser.maxDecibels = EFMedia.MAX_DB;
+          analyser.minDecibels = -90;
+          analyser.maxDecibels = -10;
+
+          const gainNode = audioContext.createGain();
+          gainNode.gain.value = this.fftGain;
+
+          const filter = audioContext.createBiquadFilter();
+          filter.type = "bandpass";
+          filter.frequency.value = 15000;
+          filter.Q.value = 0.05;
 
           const audioBufferSource = audioContext.createBufferSource();
           audioBufferSource.buffer = audioBuffer;
 
-          audioBufferSource.connect(analyser);
+          audioBufferSource.connect(filter);
+          filter.connect(gainNode);
+          gainNode.connect(analyser);
           analyser.connect(audioContext.destination);
 
           audioBufferSource.start(0, startTime, 1 / 30);
@@ -702,7 +774,7 @@ export class EFMedia extends EFTargetable(
 
         framesData.forEach((frame, frameIndex) => {
           const decayWeight = EFMedia.DECAY_WEIGHT ** frameIndex;
-          // biome-ignore lint/style/noNonNullAssertion: Will exist due to forEach
+          // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
           weightedSum += frame[i]! * decayWeight;
           weightSum += decayWeight;
         });
@@ -712,7 +784,7 @@ export class EFMedia extends EFTargetable(
 
       // Apply frequency weights using instance FREQ_WEIGHTS
       smoothedData.forEach((value, i) => {
-        // biome-ignore lint/style/noNonNullAssertion: Will exist due to forEach
+        // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
         const freqWeight = this.FREQ_WEIGHTS[i]!;
         smoothedData[i] = Math.min(255, Math.round(value * freqWeight));
       });
@@ -723,8 +795,91 @@ export class EFMedia extends EFTargetable(
         0,
         Math.floor(smoothedData.length / 2),
       );
-      this.#frequencyDataCache.set(smoothedKey, slicedData);
-      return slicedData;
+      const processedData = this.shouldInterpolateFrequencies
+        ? processFFTData(slicedData)
+        : slicedData;
+      this.#frequencyDataCache.set(smoothedKey, processedData);
+      return processedData;
     },
   });
+
+  set fftGain(value: number) {
+    const oldValue = this.fftGain;
+    this.setAttribute("fft-gain", String(value));
+    this.requestUpdate("fft-gain", oldValue);
+  }
+
+  get fftGain() {
+    return Number.parseFloat(this.getAttribute("fft-gain") ?? "3.0");
+  }
+}
+
+function processFFTData(fftData: Uint8Array, zeroThresholdPercent = 0.1) {
+  // Step 1: Determine the threshold for zeros
+  const totalBins = fftData.length;
+  const zeroThresholdCount = Math.floor(totalBins * zeroThresholdPercent);
+
+  // Step 2: Interrogate the FFT output to find the cutoff point
+  let zeroCount = 0;
+  let cutoffIndex = totalBins; // Default to the end of the array
+
+  for (let i = totalBins - 1; i >= 0; i--) {
+    // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
+    if (fftData[i]! < 10) {
+      zeroCount++;
+    } else {
+      // If we encounter a non-zero value, we can stop
+      if (zeroCount >= zeroThresholdCount) {
+        cutoffIndex = i + 1; // Include this index
+        break;
+      }
+    }
+  }
+
+  if (cutoffIndex < zeroThresholdCount) {
+    return fftData;
+  }
+
+  // Step 3: Resample the "good" portion of the data
+  const goodData = fftData.slice(0, cutoffIndex);
+  const resampledData = interpolateData(goodData, fftData.length);
+
+  // Step 4: Attenuate the top 10% of interpolated samples
+  const attenuationStartIndex = Math.floor(totalBins * 0.9);
+  for (let i = attenuationStartIndex; i < totalBins; i++) {
+    // Calculate attenuation factor that goes from 1 to 0 over the top 10%
+    const attenuationProgress =
+      (i - attenuationStartIndex) / (totalBins - attenuationStartIndex) + 0.2;
+    const attenuationFactor = Math.max(0, 1 - attenuationProgress);
+    // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
+    resampledData[i] = Math.floor(resampledData[i]! * attenuationFactor);
+  }
+
+  return resampledData;
+}
+
+function interpolateData(data: Uint8Array, targetSize: number) {
+  const resampled = new Uint8Array(targetSize);
+  const dataLength = data.length;
+
+  for (let i = 0; i < targetSize; i++) {
+    // Calculate the corresponding index in the original data
+    const ratio = (i / (targetSize - 1)) * (dataLength - 1);
+    const index = Math.floor(ratio);
+    const fraction = ratio - index;
+
+    // Handle edge cases
+    if (index >= dataLength - 1) {
+      // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
+      resampled[i] = data[dataLength - 1]!; // Last value
+    } else {
+      // Linear interpolation
+      resampled[i] = Math.round(
+        // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
+        data[index]! * (1 - fraction) + data[index + 1]! * fraction,
+      );
+    }
+  }
+
+  return resampled;
 }

package/src/elements/EFSourceMixin.ts

@@ -18,12 +18,12 @@ export function EFSourceMixin<T extends Constructor<LitElement>>(
 ) {
   class EFSourceElement extends superClass {
     get apiHost() {
-      return (
+      const apiHost =
         this.closest("ef-configuration")?.apiHost ??
         this.closest("ef-workbench")?.apiHost ??
-        this.closest("ef-preview")?.apiHost ??
-        "https://editframe.dev"
-      );
+        this.closest("ef-preview")?.apiHost;
+
+      return apiHost || "https://editframe.dev";
     }
 
     @property({ type: String })