@editframe/elements 0.15.0-beta.8 → 0.16.0-beta.0

package/dist/elements/EFMedia.js

@@ -6,11 +6,11 @@ import { property, state } from "lit/decorators.js";
 import { VideoAsset } from "@editframe/assets/EncodedAsset.js";
 import { MP4File } from "@editframe/assets/MP4File.js";
 import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
-import { EF_RENDERING } from "../EF_RENDERING.js";
 import { EFSourceMixin } from "./EFSourceMixin.js";
-import { EFTemporal, isEFTemporal } from "./EFTemporal.js";
+import { EFTemporal } from "./EFTemporal.js";
 import { FetchMixin } from "./FetchMixin.js";
 import { EFTargetable } from "./TargetController.js";
+import { updateAnimations } from "./updateAnimations.js";
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __decorateClass = (decorators, target, key, kind) => {
@@ -70,12 +70,20 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
     this.trackFragmentIndexLoader = new Task(this, {
       args: () => [this.fragmentIndexPath(), this.fetch],
       task: async ([fragmentIndexPath, fetch], { signal }) => {
-        const response = await fetch(fragmentIndexPath, { signal });
-        return await response.json();
+        try {
+          const response = await fetch(fragmentIndexPath, { signal });
+          return await response.json();
+        } catch (error) {
+          log("Failed to load track fragment index", error);
+          return void 0;
+        }
       },
       onComplete: () => {
+        this.requestUpdate("intrinsicDurationMs");
         this.requestUpdate("ownCurrentTimeMs");
+        console.log("Requesting update for durationMs", this, this.rootTimegroup);
         this.rootTimegroup?.requestUpdate("ownCurrentTimeMs");
+        this.rootTimegroup?.requestUpdate("durationMs");
       }
     });
     this.initSegmentsLoader = new Task(this, {
@@ -242,8 +250,100 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
         };
       }
     });
-    this.fftSize = 512;
-    this.fftDecay = 8;
+    this.#byteTimeDomainCache = new LRUCache(100);
+    this.byteTimeDomainTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [
+        this.audioBufferTask.status,
+        this.currentSourceTimeMs,
+        this.fftSize,
+        this.fftDecay,
+        this.fftGain,
+        this.shouldInterpolateFrequencies
+      ],
+      task: async () => {
+        await this.audioBufferTask.taskComplete;
+        if (!this.audioBufferTask.value) return null;
+        if (this.currentSourceTimeMs <= 0) return null;
+        const currentTimeMs = this.currentSourceTimeMs;
+        const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
+        const audioBuffer = this.audioBufferTask.value.buffer;
+        const smoothedKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftDecay}:${this.fftGain}:${startOffsetMs}:${currentTimeMs}`;
+        const cachedData = this.#byteTimeDomainCache.get(smoothedKey);
+        if (cachedData) return cachedData;
+        const framesData = await Promise.all(
+          Array.from({ length: this.fftDecay }, async (_, frameIndex) => {
+            const frameOffset = frameIndex * (1e3 / 30);
+            const startTime = Math.max(
+              0,
+              (currentTimeMs - frameOffset - startOffsetMs) / 1e3
+            );
+            const cacheKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftGain}:${startOffsetMs}:${startTime}`;
+            const cachedFrame = this.#byteTimeDomainCache.get(cacheKey);
+            if (cachedFrame) return cachedFrame;
+            const audioContext = new OfflineAudioContext(
+              2,
+              48e3 * (1 / 30),
+              48e3
+            );
+            const source = audioContext.createBufferSource();
+            source.buffer = audioBuffer;
+            const analyser = audioContext.createAnalyser();
+            analyser.fftSize = this.fftSize;
+            analyser.minDecibels = -90;
+            analyser.maxDecibels = -20;
+            const gainNode = audioContext.createGain();
+            gainNode.gain.value = this.fftGain;
+            source.connect(gainNode);
+            gainNode.connect(analyser);
+            analyser.connect(audioContext.destination);
+            source.start(0, startTime, 1 / 30);
+            const dataLength = analyser.fftSize / 2;
+            try {
+              await audioContext.startRendering();
+              const frameData = new Uint8Array(dataLength);
+              analyser.getByteTimeDomainData(frameData);
+              const points = new Uint8Array(dataLength);
+              for (let i = 0; i < dataLength; i++) {
+                const pointSamples = frameData.slice(
+                  i * (frameData.length / dataLength),
+                  (i + 1) * (frameData.length / dataLength)
+                );
+                const rms = Math.sqrt(
+                  pointSamples.reduce((sum, sample) => {
+                    const normalized = (sample - 128) / 128;
+                    return sum + normalized * normalized;
+                  }, 0) / pointSamples.length
+                );
+                const avgSign = Math.sign(
+                  pointSamples.reduce((sum, sample) => sum + (sample - 128), 0)
+                );
+                points[i] = Math.min(255, Math.round(128 + avgSign * rms * 128));
+              }
+              this.#byteTimeDomainCache.set(cacheKey, points);
+              return points;
+            } finally {
+              source.disconnect();
+              analyser.disconnect();
+            }
+          })
+        );
+        const frameLength = framesData[0]?.length ?? 0;
+        const smoothedData = new Uint8Array(frameLength);
+        for (let i = 0; i < frameLength; i++) {
+          let weightedSum = 0;
+          let weightSum = 0;
+          framesData.forEach((frame, frameIndex) => {
+            const decayWeight = _EFMedia2.DECAY_WEIGHT ** frameIndex;
+            weightedSum += (frame[i] ?? 0) * decayWeight;
+            weightSum += decayWeight;
+          });
+          smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+        }
+        this.#byteTimeDomainCache.set(smoothedKey, smoothedData);
+        return smoothedData;
+      }
+    });
     this.#frequencyDataCache = new LRUCache(100);
     this.frequencyDataTask = new Task(this, {
       autoRun: EF_INTERACTIVE,
@@ -251,9 +351,9 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
         this.audioBufferTask.status,
         this.currentSourceTimeMs,
         this.fftSize,
-        // Add fftSize to dependency array
-        this.fftDecay
-        // Add fftDecay to dependency array
+        this.fftDecay,
+        this.fftGain,
+        this.shouldInterpolateFrequencies
       ],
       task: async () => {
         await this.audioBufferTask.taskComplete;
@@ -262,7 +362,7 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
         const currentTimeMs = this.currentSourceTimeMs;
         const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
         const audioBuffer = this.audioBufferTask.value.buffer;
-        const smoothedKey = `${this.fftSize}:${this.fftDecay}:${startOffsetMs}:${currentTimeMs}`;
+        const smoothedKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftDecay}:${this.fftGain}:${startOffsetMs}:${currentTimeMs}`;
         const cachedSmoothedData = this.#frequencyDataCache.get(smoothedKey);
         if (cachedSmoothedData) {
           return cachedSmoothedData;
@@ -274,7 +374,7 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
               0,
               (currentTimeMs - frameOffset - startOffsetMs) / 1e3
             );
-            const cacheKey = `${this.fftSize}:${startOffsetMs}:${startTime}`;
+            const cacheKey = `${this.shouldInterpolateFrequencies}:${this.fftSize}:${this.fftGain}:${startOffsetMs}:${startTime}`;
             const cachedFrame = this.#frequencyDataCache.get(cacheKey);
             if (cachedFrame) {
               return cachedFrame;
@@ -286,11 +386,19 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
             );
             const analyser = audioContext.createAnalyser();
             analyser.fftSize = this.fftSize;
-            analyser.minDecibels = _EFMedia2.MIN_DB;
-            analyser.maxDecibels = _EFMedia2.MAX_DB;
+            analyser.minDecibels = -90;
+            analyser.maxDecibels = -10;
+            const gainNode = audioContext.createGain();
+            gainNode.gain.value = this.fftGain;
+            const filter = audioContext.createBiquadFilter();
+            filter.type = "bandpass";
+            filter.frequency.value = 15e3;
+            filter.Q.value = 0.05;
             const audioBufferSource = audioContext.createBufferSource();
             audioBufferSource.buffer = audioBuffer;
-            audioBufferSource.connect(analyser);
+            audioBufferSource.connect(filter);
+            filter.connect(gainNode);
+            gainNode.connect(analyser);
             analyser.connect(audioContext.destination);
             audioBufferSource.start(0, startTime, 1 / 30);
             try {
@@ -325,8 +433,9 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
           0,
           Math.floor(smoothedData.length / 2)
         );
-        this.#frequencyDataCache.set(smoothedKey, slicedData);
-        return slicedData;
+        const processedData = this.shouldInterpolateFrequencies ? processFFTData(slicedData) : slicedData;
+        this.#frequencyDataCache.set(smoothedKey, processedData);
+        return processedData;
       }
     });
   }
@@ -350,18 +459,12 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
   }
   fragmentIndexPath() {
     if (this.assetId) {
-      if (EF_RENDERING()) {
-        return `editframe://api/v1/isobmff_files/${this.assetId}/index`;
-      }
       return `${this.apiHost}/api/v1/isobmff_files/${this.assetId}/index`;
     }
     return `/@ef-track-fragment-index/${this.src ?? ""}`;
   }
   fragmentTrackPath(trackId) {
     if (this.assetId) {
-      if (EF_RENDERING()) {
-        return `editframe://api/v1/isobmff_tracks/${this.assetId}/${trackId}`;
-      }
       return `${this.apiHost}/api/v1/isobmff_tracks/${this.assetId}/${trackId}`;
     }
     return `/@ef-track/${this.src ?? ""}?trackId=${trackId}`;
@@ -384,71 +487,13 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
       this.executeSeek(this.currentSourceTimeMs);
     }
     if (changedProperties.has("currentTime") || changedProperties.has("ownCurrentTimeMs")) {
-      const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
-      if (this.startTimeMs > timelineTimeMs || this.endTimeMs < timelineTimeMs) {
-        this.style.display = "none";
-        return;
-      }
-      this.style.display = "";
-      const animations = this.getAnimations({ subtree: true });
-      this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
-      this.style.setProperty(
-        "--ef-transition-duration",
-        `${this.parentTimegroup?.overlapMs ?? 0}ms`
-      );
-      this.style.setProperty(
-        "--ef-transition-out-start",
-        `${this.durationMs - (this.parentTimegroup?.overlapMs ?? 0)}ms`
-      );
-      for (const animation of animations) {
-        if (animation.playState === "running") {
-          animation.pause();
-        }
-        const effect = animation.effect;
-        if (!(effect && effect instanceof KeyframeEffect)) {
-          return;
-        }
-        const target = effect.target;
-        if (!target) {
-          return;
-        }
-        if (target.closest("ef-video, ef-audio") !== this) {
-          return;
-        }
-        if (isEFTemporal(target)) {
-          const timing = effect.getTiming();
-          const duration = Number(timing.duration) ?? 0;
-          const delay = Number(timing.delay);
-          const newTime = Math.floor(
-            Math.min(target.ownCurrentTimeMs, duration - 1 + delay)
-          );
-          if (Number.isNaN(newTime)) {
-            return;
-          }
-          animation.currentTime = newTime;
-        } else if (target) {
-          const nearestTimegroup = target.closest("ef-timegroup");
-          if (!nearestTimegroup) {
-            return;
-          }
-          const timing = effect.getTiming();
-          const duration = Number(timing.duration) ?? 0;
-          const delay = Number(timing.delay);
-          const newTime = Math.floor(
-            Math.min(nearestTimegroup.ownCurrentTimeMs, duration - 1 + delay)
-          );
-          if (Number.isNaN(newTime)) {
-            return;
-          }
-          animation.currentTime = newTime;
-        }
-      }
+      updateAnimations(this);
     }
   }
   get hasOwnDuration() {
     return true;
   }
-  get durationMs() {
+  get intrinsicDurationMs() {
     if (!this.trackFragmentIndexLoader.value) {
       return 0;
     }
@@ -460,30 +505,18 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
     if (durations.length === 0) {
       return 0;
     }
-    if (this.sourceInMs && this.sourceOutMs && this.sourceOutMs > this.sourceInMs) {
-      return Math.max(this.sourceOutMs - this.sourceInMs);
-    }
-    if (this.sourceInMs) {
-      return Math.max(...durations) - this.trimStartMs - this.trimEndMs - this.sourceInMs;
-    }
-    if (this.sourceOutMs) {
-      return Math.max(...durations) - this.trimStartMs - this.trimEndMs - this.sourceOutMs;
-    }
-    if (this.sourceInMs && this.sourceOutMs) {
-      return Math.max(...durations) - this.trimStartMs - this.trimEndMs - this.sourceOutMs - this.sourceInMs;
-    }
-    return Math.max(...durations) - this.trimStartMs - this.trimEndMs;
+    return Math.max(...durations);
   }
   #audioContext;
   async fetchAudioSpanningTime(fromMs, toMs) {
     if (this.sourceInMs) {
-      fromMs -= this.startTimeMs - this.trimStartMs - this.sourceInMs;
+      fromMs -= this.startTimeMs - (this.trimStartMs ?? 0) - (this.sourceInMs ?? 0);
     }
     if (this.sourceOutMs) {
-      toMs -= this.startTimeMs - this.trimStartMs - this.sourceOutMs;
+      toMs -= this.startTimeMs - (this.trimStartMs ?? 0) - (this.sourceOutMs ?? 0);
     }
-    fromMs -= this.startTimeMs - this.trimStartMs;
-    toMs -= this.startTimeMs - this.trimStartMs;
+    fromMs -= this.startTimeMs - (this.trimStartMs ?? 0);
+    toMs -= this.startTimeMs - (this.trimStartMs ?? 0);
     await this.trackFragmentIndexLoader.taskComplete;
     const audioTrackId = this.defaultAudioTrackId;
     if (!audioTrackId) {
@@ -537,15 +570,39 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
     });
     return {
       blob: audioBlob,
-      startMs: firstFragment.dts / audioTrackIndex.timescale * 1e3 - this.trimStartMs,
-      endMs: lastFragment.dts / audioTrackIndex.timescale * 1e3 + lastFragment.duration / audioTrackIndex.timescale * 1e3 - this.trimEndMs
+      startMs: firstFragment.dts / audioTrackIndex.timescale * 1e3 - (this.trimStartMs ?? 0),
+      endMs: lastFragment.dts / audioTrackIndex.timescale * 1e3 + lastFragment.duration / audioTrackIndex.timescale * 1e3 - (this.trimEndMs ?? 0)
     };
   }
-  static {
-    this.MIN_DB = -90;
+  set fftSize(value) {
+    const oldValue = this.fftSize;
+    this.setAttribute("fft-size", String(value));
+    this.requestUpdate("fft-size", oldValue);
   }
-  static {
-    this.MAX_DB = -20;
+  set fftDecay(value) {
+    const oldValue = this.fftDecay;
+    this.setAttribute("fft-decay", String(value));
+    this.requestUpdate("fft-decay", oldValue);
+  }
+  get fftSize() {
+    return Number.parseInt(this.getAttribute("fft-size") ?? "128", 10);
+  }
+  get fftDecay() {
+    return Number.parseInt(this.getAttribute("fft-decay") ?? "8", 10);
+  }
+  set interpolateFrequencies(value) {
+    const oldValue = this.interpolateFrequencies;
+    this.setAttribute("interpolate-frequencies", String(value));
+    this.requestUpdate("interpolate-frequencies", oldValue);
+  }
+  get interpolateFrequencies() {
+    return this.getAttribute("interpolate-frequencies") !== "false";
+  }
+  get shouldInterpolateFrequencies() {
+    if (this.hasAttribute("interpolate-frequencies")) {
+      return this.getAttribute("interpolate-frequencies") !== "false";
+    }
+    return false;
   }
   static {
     this.DECAY_WEIGHT = 0.7;
@@ -568,7 +625,16 @@ const _EFMedia = class _EFMedia2 extends EFTargetable(
     freqWeightsCache.set(this.fftSize, weights);
     return weights;
   }
+  #byteTimeDomainCache;
   #frequencyDataCache;
+  set fftGain(value) {
+    const oldValue = this.fftGain;
+    this.setAttribute("fft-gain", String(value));
+    this.requestUpdate("fft-gain", oldValue);
+  }
+  get fftGain() {
+    return Number.parseFloat(this.getAttribute("fft-gain") ?? "3.0");
+  }
 };
 __decorateClass([
   property({ type: Number })
@@ -579,13 +645,53 @@ __decorateClass([
 __decorateClass([
   state()
 ], _EFMedia.prototype, "desiredSeekTimeMs", 2);
-__decorateClass([
-  property({ type: Number })
-], _EFMedia.prototype, "fftSize", 2);
-__decorateClass([
-  property({ type: Number })
-], _EFMedia.prototype, "fftDecay", 2);
 let EFMedia = _EFMedia;
+function processFFTData(fftData, zeroThresholdPercent = 0.1) {
+  const totalBins = fftData.length;
+  const zeroThresholdCount = Math.floor(totalBins * zeroThresholdPercent);
+  let zeroCount = 0;
+  let cutoffIndex = totalBins;
+  for (let i = totalBins - 1; i >= 0; i--) {
+    if (fftData[i] < 10) {
+      zeroCount++;
+    } else {
+      if (zeroCount >= zeroThresholdCount) {
+        cutoffIndex = i + 1;
+        break;
+      }
+    }
+  }
+  if (cutoffIndex < zeroThresholdCount) {
+    return fftData;
+  }
+  const goodData = fftData.slice(0, cutoffIndex);
+  const resampledData = interpolateData(goodData, fftData.length);
+  const attenuationStartIndex = Math.floor(totalBins * 0.9);
+  for (let i = attenuationStartIndex; i < totalBins; i++) {
+    const attenuationProgress = (i - attenuationStartIndex) / (totalBins - attenuationStartIndex) + 0.2;
+    const attenuationFactor = Math.max(0, 1 - attenuationProgress);
+    resampledData[i] = Math.floor(resampledData[i] * attenuationFactor);
+  }
+  return resampledData;
+}
+function interpolateData(data, targetSize) {
+  const resampled = new Uint8Array(targetSize);
+  const dataLength = data.length;
+  for (let i = 0; i < targetSize; i++) {
+    const ratio = i / (targetSize - 1) * (dataLength - 1);
+    const index = Math.floor(ratio);
+    const fraction = ratio - index;
+    if (index >= dataLength - 1) {
+      resampled[i] = data[dataLength - 1];
+    } else {
+      resampled[i] = Math.round(
+        // biome-ignore lint/style/noNonNullAssertion: Manual bounds check
+        data[index] * (1 - fraction) + data[index + 1] * fraction
+      );
+    }
+  }
+  return resampled;
+}
 export {
   EFMedia,
   deepGetMediaElements

package/dist/elements/EFSourceMixin.js

@@ -25,7 +25,8 @@ function EFSourceMixin(superClass, options) {
       });
     }
     get apiHost() {
-      return this.closest("ef-configuration")?.apiHost ?? this.closest("ef-workbench")?.apiHost ?? this.closest("ef-preview")?.apiHost ?? "https://editframe.dev";
+      const apiHost = this.closest("ef-configuration")?.apiHost ?? this.closest("ef-workbench")?.apiHost ?? this.closest("ef-preview")?.apiHost;
+      return apiHost || "https://editframe.dev";
     }
     productionSrc() {
       if (!this.md5SumLoader.value) {

package/dist/elements/EFTemporal.browsertest.d.ts

@@ -1,10 +1,11 @@
 import { LitElement } from 'lit';
-declare const TestTemporal_base: (new (...args: any[]) => import('./EFTemporal.js').TemporalMixinInterface) & typeof LitElement;
-declare class TestTemporal extends TestTemporal_base {
+declare const TenSeconds_base: (new (...args: any[]) => import('./EFTemporal.js').TemporalMixinInterface) & typeof LitElement;
+declare class TenSeconds extends TenSeconds_base {
+    get intrinsicDurationMs(): number;
 }
 declare global {
     interface HTMLElementTagNameMap {
-        "test-temporal": TestTemporal;
+        "ten-seconds": TenSeconds;
     }
 }
 export {};

package/dist/elements/EFTemporal.d.ts

@@ -10,6 +10,7 @@ export declare class TemporalMixinInterface {
      * Whether the element has a duration set as an attribute.
      */
     get hasExplicitDuration(): boolean;
+    get sourceStartMs(): number;
     /**
      * Used to trim the start of the media.
      *
@@ -19,7 +20,7 @@ export declare class TemporalMixinInterface {
      *
      * @domAttribute "trimstart"
      */
-    get trimStartMs(): number;
+    get trimStartMs(): number | undefined;
     /**
      * Used to trim the end of the media.
      *
@@ -30,10 +31,10 @@ export declare class TemporalMixinInterface {
      * @domAttribute "trimend"
      */
     get trimEndMs(): number;
-    set trimStartMs(value: number);
-    set trimEndMs(value: number);
-    set trimstart(value: string);
-    set trimend(value: string);
+    set trimStartMs(value: number | undefined);
+    set trimEndMs(value: number | undefined);
+    set trimstart(value: string | undefined);
+    set trimend(value: string | undefined);
     /**
      * The source in time of the element.
      *
@@ -53,7 +54,7 @@ export declare class TemporalMixinInterface {
      *
      * @domAttribute "sourcein"
      */
-    get sourceInMs(): number;
+    get sourceInMs(): number | undefined;
     /**
      * The source out time of the element.
      *
@@ -75,15 +76,17 @@ export declare class TemporalMixinInterface {
      *
      * @domAttribute "sourceout"
      */
-    get sourceOutMs(): number;
-    set sourceInMs(value: number);
-    set sourceOutMs(value: number);
-    set sourcein(value: string);
-    set sourceout(value: string);
+    get sourceOutMs(): number | undefined;
+    set sourceInMs(value: number | undefined);
+    set sourceOutMs(value: number | undefined);
+    set sourcein(value: string | undefined);
+    set sourceout(value: string | undefined);
     /**
      * @domAttribute "duration"
      */
     get durationMs(): number;
+    get explicitDurationMs(): number | undefined;
+    get intrinsicDurationMs(): number | undefined;
     /**
      * The start time of the element within its root timegroup in milliseconds.
      *