@editframe/elements 0.15.0-beta.1 → 0.15.0-beta.11

package/dist/EF_FRAMEGEN.js

@@ -104,8 +104,6 @@ class EfFramegen {
     this.audioBufferPromise = firstGroup.renderAudio(
       renderOptions.encoderOptions.alignedFromUs / 1e3,
       renderOptions.encoderOptions.alignedToUs / 1e3
-      // renderOptions.encoderOptions.fromMs,
-      // renderOptions.encoderOptions.toMs,
     );
     console.log("Initialized");
   }

package/dist/elements/EFAudio.d.ts

@@ -2,7 +2,6 @@ import { Task } from '@lit/task';
 import { EFMedia } from './EFMedia.js';
 export declare class EFAudio extends EFMedia {
     audioElementRef: import('lit-html/directives/ref.js').Ref<HTMLAudioElement>;
-    src: string;
     render(): import('lit-html').TemplateResult<1>;
     get audioElement(): HTMLAudioElement | undefined;
     frameTask: Task<readonly [import('@lit/task').TaskStatus, import('@lit/task').TaskStatus, import('@lit/task').TaskStatus, import('@lit/task').TaskStatus, import('@lit/task').TaskStatus], void>;

package/dist/elements/EFAudio.js

@@ -1,6 +1,6 @@
 import { Task } from "@lit/task";
 import { html } from "lit";
-import { property, customElement } from "lit/decorators.js";
+import { customElement } from "lit/decorators.js";
 import { createRef, ref } from "lit/directives/ref.js";
 import { EFMedia } from "./EFMedia.js";
 var __defProp = Object.defineProperty;
@@ -17,7 +17,6 @@ let EFAudio = class extends EFMedia {
   constructor() {
     super(...arguments);
     this.audioElementRef = createRef();
-    this.src = "";
     this.frameTask = new Task(this, {
       args: () => [
         this.trackFragmentIndexLoader.status,
@@ -43,9 +42,6 @@ let EFAudio = class extends EFMedia {
     return this.audioElementRef.value;
   }
 };
-__decorateClass([
-  property({ type: String })
-], EFAudio.prototype, "src", 2);
 EFAudio = __decorateClass([
   customElement("ef-audio")
 ], EFAudio);

package/dist/elements/EFCaptions.js

@@ -335,7 +335,7 @@ let EFCaptions = class extends EFSourceMixin(
     if (!transcriptionFragment) {
       return;
     }
-    const currentTimeMs = this.targetElement.trimAdjustedOwnCurrentTimeMs;
+    const currentTimeMs = this.targetElement.currentSourceTimeMs;
     const currentTimeSec = currentTimeMs / 1e3;
     const currentWord = transcriptionFragment.word_segments.find(
       (word) => currentTimeSec >= word.start && currentTimeSec <= word.end

package/dist/elements/EFMedia.d.ts

@@ -70,6 +70,7 @@ export declare class EFMedia extends EFMedia_base {
     private static readonly MAX_DB;
     private static readonly DECAY_WEIGHT;
     get FREQ_WEIGHTS(): Float32Array;
-    frequencyDataTask: Task<readonly [import('@lit/task').TaskStatus, number], Uint8Array | null>;
+    byteTimeDomainTask: Task<readonly [import('@lit/task').TaskStatus, number, number, number], Uint8Array | null>;
+    frequencyDataTask: Task<readonly [import('@lit/task').TaskStatus, number, number, number], Uint8Array | null>;
 }
 export {};

package/dist/elements/EFMedia.js

@@ -10,6 +10,7 @@ import { EF_RENDERING } from "../EF_RENDERING.js";
 import { EFSourceMixin } from "./EFSourceMixin.js";
 import { EFTemporal, isEFTemporal } from "./EFTemporal.js";
 import { FetchMixin } from "./FetchMixin.js";
+import { EFTargetable } from "./TargetController.js";
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __decorateClass = (decorators, target, key, kind) => {
@@ -40,12 +41,13 @@ class LRUCache {
       this.cache.delete(key);
     } else if (this.cache.size >= this.maxSize) {
       const firstKey = this.cache.keys().next().value;
-      this.cache.delete(firstKey);
+      if (firstKey) {
+        this.cache.delete(firstKey);
+      }
     }
     this.cache.set(key, value);
   }
 }
-const frequencyDataCache = new LRUCache(100);
 const deepGetMediaElements = (element, medias = []) => {
   for (const child of Array.from(element.children)) {
     if (child instanceof EFMedia) {
@@ -56,9 +58,11 @@ const deepGetMediaElements = (element, medias = []) => {
   }
   return medias;
 };
-const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
-  assetType: "isobmff_files"
-}) {
+const _EFMedia = class _EFMedia2 extends EFTargetable(
+  EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
+    assetType: "isobmff_files"
+  })
+) {
   constructor() {
     super(...arguments);
     this.currentTimeMs = 0;
@@ -240,16 +244,116 @@ const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(Lit
     });
     this.fftSize = 512;
     this.fftDecay = 8;
+    this.#byteTimeDomainCache = new LRUCache(100);
+    this.byteTimeDomainTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [
+        this.audioBufferTask.status,
+        this.currentSourceTimeMs,
+        this.fftSize,
+        this.fftDecay
+      ],
+      task: async () => {
+        await this.audioBufferTask.taskComplete;
+        if (!this.audioBufferTask.value) return null;
+        if (this.currentSourceTimeMs <= 0) return null;
+        const currentTimeMs = this.currentSourceTimeMs;
+        const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
+        const audioBuffer = this.audioBufferTask.value.buffer;
+        const smoothedKey = `${this.fftSize}:${this.fftDecay}:${startOffsetMs}:${currentTimeMs}`;
+        const cachedSmoothedData = this.#byteTimeDomainCache.get(smoothedKey);
+        if (cachedSmoothedData) {
+          return cachedSmoothedData;
+        }
+        const framesData = await Promise.all(
+          Array.from({ length: this.fftDecay }, async (_, i) => {
+            const frameOffset = i * (1e3 / 30);
+            const startTime = Math.max(
+              0,
+              (currentTimeMs - frameOffset - startOffsetMs) / 1e3
+            );
+            const cacheKey = `${this.fftSize}:${startOffsetMs}:${startTime}`;
+            const cachedFrame = this.#byteTimeDomainCache.get(cacheKey);
+            if (cachedFrame) {
+              return cachedFrame;
+            }
+            const audioContext = new OfflineAudioContext(
+              2,
+              48e3 * (1 / 30),
+              48e3
+            );
+            const analyser = audioContext.createAnalyser();
+            analyser.fftSize = this.fftSize;
+            const gainNode = audioContext.createGain();
+            gainNode.gain.value = 10;
+            analyser.smoothingTimeConstant = 0.4;
+            analyser.minDecibels = -90;
+            analyser.maxDecibels = -10;
+            const audioBufferSource = audioContext.createBufferSource();
+            audioBufferSource.buffer = audioBuffer;
+            const filter = audioContext.createBiquadFilter();
+            filter.type = "bandpass";
+            filter.frequency.value = 1e3;
+            filter.Q.value = 0.5;
+            audioBufferSource.connect(gainNode);
+            gainNode.connect(filter);
+            filter.connect(analyser);
+            analyser.connect(audioContext.destination);
+            audioBufferSource.start(0, startTime, 1 / 30);
+            try {
+              await audioContext.startRendering();
+              const frameData = new Uint8Array(analyser.fftSize);
+              analyser.getByteTimeDomainData(frameData);
+              this.#byteTimeDomainCache.set(cacheKey, frameData);
+              return frameData;
+            } finally {
+              audioBufferSource.disconnect();
+              analyser.disconnect();
+            }
+          })
+        );
+        const frameLength = framesData[0]?.length ?? 0;
+        const smoothedData = new Uint8Array(frameLength);
+        for (let i = 0; i < frameLength; i++) {
+          let weightedSum = 0;
+          let weightSum = 0;
+          framesData.forEach((frame, frameIndex) => {
+            const decayWeight = _EFMedia2.DECAY_WEIGHT ** frameIndex;
+            weightedSum += frame[i] * decayWeight;
+            weightSum += decayWeight;
+          });
+          smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+        }
+        this.#byteTimeDomainCache.set(
+          smoothedKey,
+          smoothedData.slice(0, Math.floor(smoothedData.length * 0.8))
+        );
+        return smoothedData;
+      }
+    });
+    this.#frequencyDataCache = new LRUCache(100);
     this.frequencyDataTask = new Task(this, {
       autoRun: EF_INTERACTIVE,
-      args: () => [this.audioBufferTask.status, this.trimAdjustedOwnCurrentTimeMs],
+      args: () => [
+        this.audioBufferTask.status,
+        this.currentSourceTimeMs,
+        this.fftSize,
+        // Add fftSize to dependency array
+        this.fftDecay
+        // Add fftDecay to dependency array
+      ],
       task: async () => {
         await this.audioBufferTask.taskComplete;
         if (!this.audioBufferTask.value) return null;
-        if (this.trimAdjustedOwnCurrentTimeMs <= 0) return null;
-        const currentTimeMs = this.trimAdjustedOwnCurrentTimeMs;
+        if (this.currentSourceTimeMs <= 0) return null;
+        const currentTimeMs = this.currentSourceTimeMs;
         const startOffsetMs = this.audioBufferTask.value.startOffsetMs;
         const audioBuffer = this.audioBufferTask.value.buffer;
+        const smoothedKey = `${this.fftSize}:${this.fftDecay}:${startOffsetMs}:${currentTimeMs}`;
+        const cachedSmoothedData = this.#frequencyDataCache.get(smoothedKey);
+        if (cachedSmoothedData) {
+          return cachedSmoothedData;
+        }
         const framesData = await Promise.all(
           Array.from({ length: this.fftDecay }, async (_, i) => {
             const frameOffset = i * (1e3 / 30);
@@ -257,8 +361,8 @@ const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(Lit
               0,
               (currentTimeMs - frameOffset - startOffsetMs) / 1e3
             );
-            const cacheKey = `${startOffsetMs},${startTime}`;
-            const cachedFrame = frequencyDataCache.get(cacheKey);
+            const cacheKey = `${this.fftSize}:${startOffsetMs}:${startTime}`;
+            const cachedFrame = this.#frequencyDataCache.get(cacheKey);
             if (cachedFrame) {
               return cachedFrame;
             }
@@ -278,9 +382,9 @@ const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(Lit
             audioBufferSource.start(0, startTime, 1 / 30);
             try {
               await audioContext.startRendering();
-              const frameData = new Uint8Array(analyser.frequencyBinCount);
+              const frameData = new Uint8Array(this.fftSize / 2);
               analyser.getByteFrequencyData(frameData);
-              frequencyDataCache.set(cacheKey, frameData);
+              this.#frequencyDataCache.set(cacheKey, frameData);
               return frameData;
             } finally {
               audioBufferSource.disconnect();
@@ -304,7 +408,12 @@ const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(Lit
           const freqWeight = this.FREQ_WEIGHTS[i];
           smoothedData[i] = Math.min(255, Math.round(value * freqWeight));
         });
-        return smoothedData;
+        const slicedData = smoothedData.slice(
+          0,
+          Math.floor(smoothedData.length / 2)
+        );
+        this.#frequencyDataCache.set(smoothedKey, slicedData);
+        return slicedData;
       }
     });
   }
@@ -359,7 +468,7 @@ const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(Lit
   }
   updated(changedProperties) {
     if (changedProperties.has("ownCurrentTimeMs")) {
-      this.executeSeek(this.trimAdjustedOwnCurrentTimeMs);
+      this.executeSeek(this.currentSourceTimeMs);
     }
     if (changedProperties.has("currentTime") || changedProperties.has("ownCurrentTimeMs")) {
       const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
@@ -546,6 +655,8 @@ const _EFMedia = class _EFMedia2 extends EFSourceMixin(EFTemporal(FetchMixin(Lit
     freqWeightsCache.set(this.fftSize, weights);
     return weights;
   }
+  #byteTimeDomainCache;
+  #frequencyDataCache;
 };
 __decorateClass([
   property({ type: Number })

package/dist/elements/EFTemporal.d.ts

@@ -135,18 +135,18 @@ export declare class TemporalMixinInterface {
      * elements.
      *
      * For example, if the media has a `sourcein` value of 10s, when `ownCurrentTimeMs` is 0s,
-     * `trimAdjustedOwnCurrentTimeMs` will be 10s.
+     * `currentSourceTimeMs` will be 10s.
      *
      *            sourcein=10s     sourceout=10s
      * /         /                /
      * |--------|=================|---------|
      *          ^
      *          |_
-     *            trimAdjustedOwnCurrentTimeMs === 10s
+     *            currentSourceTimeMs === 10s
      *          |_
      *            ownCurrentTimeMs === 0s
      */
-    get trimAdjustedOwnCurrentTimeMs(): number;
+    get currentSourceTimeMs(): number;
     set duration(value: string);
     get duration(): string;
     /**

package/dist/elements/EFTemporal.js

@@ -278,6 +278,10 @@ const EFTemporal = (superClass) => {
     get endTimeMs() {
       return this.startTimeMs + this.durationMs;
     }
+    /**
+     * The current time of the element within itself.
+     * Compare with `currentTimeMs` to see the current time with respect to the root timegroup
+     */
     get ownCurrentTimeMs() {
       if (this.rootTimegroup) {
         return Math.min(
@@ -291,7 +295,7 @@ const EFTemporal = (superClass) => {
      * Used to calculate the internal currentTimeMs of the element. This is useful
      * for mapping to internal media time codes for audio/video elements.
      */
-    get trimAdjustedOwnCurrentTimeMs() {
+    get currentSourceTimeMs() {
       if (this.rootTimegroup) {
         if (this.sourceInMs && this.sourceOutMs) {
           return Math.min(
@@ -316,7 +320,7 @@ const EFTemporal = (superClass) => {
       super.updated(changedProperties);
       if (changedProperties.has("currentTime") || changedProperties.has("ownCurrentTimeMs")) {
         const timelineTimeMs = (this.rootTimegroup ?? this).ownCurrentTimeMs;
-        if (this.startTimeMs >= timelineTimeMs || this.endTimeMs <= timelineTimeMs) {
+        if (this.startTimeMs > timelineTimeMs || this.endTimeMs < timelineTimeMs) {
           this.style.display = "none";
           return;
         }

package/dist/elements/EFTimegroup.d.ts

@@ -27,11 +27,7 @@ export declare class EFTimegroup extends EFTimegroup_base {
      * that caused issues with constructing audio data. We had negative durations
      * in calculations and it was not clear why.
      */
-    waitForMediaDurations(): Promise<({
-        trackId: string;
-        buffer: import('mp4box').MP4ArrayBuffer;
-        mp4File: import('../../../assets/src/MP4File.ts').MP4File;
-    }[] | undefined)[]>;
+    waitForMediaDurations(): Promise<Record<number, import('../../../assets/src/index.ts').TrackFragmentIndex>[]>;
     get childTemporals(): import('./EFTemporal.js').TemporalMixinInterface[];
     protected updated(changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>): void;
     private updateAnimations;

package/dist/elements/EFTimegroup.js

@@ -45,7 +45,7 @@ let EFTimegroup = class extends EFTemporal(LitElement) {
     __privateAdd(this, _EFTimegroup_instances);
     this._timeGroupContext = this;
     __privateAdd(this, _currentTime, 0);
-    this.mode = "sequence";
+    this.mode = "contain";
     this.overlapMs = 0;
     this.fit = "none";
     __privateAdd(this, _resizeObserver);
@@ -186,10 +186,9 @@ let EFTimegroup = class extends EFTemporal(LitElement) {
    * in calculations and it was not clear why.
    */
   async waitForMediaDurations() {
+    const mediaElements = deepGetMediaElements(this);
     return await Promise.all(
-      deepGetMediaElements(this).map(
-        (media) => media.initSegmentsLoader.taskComplete
-      )
+      mediaElements.map((m) => m.trackFragmentIndexLoader.taskComplete)
     );
   }
   get childTemporals() {
@@ -402,7 +401,7 @@ __decorateClass([
   property({ type: String })
 ], EFTimegroup.prototype, "fit", 2);
 __decorateClass([
-  property({ type: Number })
+  property({ type: Number, attribute: "currenttime" })
 ], EFTimegroup.prototype, "currentTime", 1);
 EFTimegroup = __decorateClass([
   customElement("ef-timegroup")

package/dist/elements/EFWaveform.d.ts

@@ -1,8 +1,9 @@
-import { EFAudio } from './EFAudio.js';
 import { Task } from '@lit/task';
 import { LitElement, PropertyValueMap } from 'lit';
 import { Ref } from 'lit/directives/ref.js';
+import { EFAudio } from './EFAudio.js';
 import { EFVideo } from './EFVideo.js';
+import { TargetController } from './TargetController.ts';
 declare const EFWaveform_base: (new (...args: any[]) => import('./EFTemporal.js').TemporalMixinInterface) & typeof LitElement;
 export declare class EFWaveform extends EFWaveform_base {
     static styles: import('lit').CSSResult;
@@ -12,11 +13,12 @@ export declare class EFWaveform extends EFWaveform_base {
     private resizeObserver?;
     private mutationObserver?;
     render(): import('lit-html').TemplateResult<1>;
-    mode: "roundBars" | "bars" | "bricks" | "line" | "pixel" | "wave";
+    mode: "roundBars" | "bars" | "bricks" | "line" | "curve" | "pixel" | "wave" | "spikes";
     color: string;
-    targetSelector: string;
+    target: string;
+    targetElement: EFAudio | EFVideo | null;
     lineWidth: number;
-    set target(value: string);
+    targetController: TargetController;
     connectedCallback(): void;
     disconnectedCallback(): void;
     private resizeCanvas;
@@ -26,11 +28,17 @@ export declare class EFWaveform extends EFWaveform_base {
     protected drawRoundBars(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
     protected drawEqualizer(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
     protected drawLine(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
+    protected drawCurve(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
     protected drawPixel(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
     protected drawWave(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
-    frameTask: Task<readonly [import('@lit/task').TaskStatus | undefined], void>;
+    protected drawSpikes(ctx: CanvasRenderingContext2D, frequencyData: Uint8Array): void;
+    frameTask: Task<readonly [EFAudio | EFVideo | null, Uint8Array | null | undefined], void>;
     get durationMs(): number;
-    get targetElement(): EFAudio | EFVideo | null;
     protected updated(changedProperties: PropertyValueMap<this>): void;
 }
+declare global {
+    interface HTMLElementTagNameMap {
+        "ef-waveform": EFWaveform & Element;
+    }
+}
 export {};