@editframe/elements 0.26.3-beta.0 → 0.30.0-beta.13

package/src/elements/EFMedia.ts

@@ -1,341 +0,0 @@
-import { provide } from "@lit/context";
-import { css, LitElement, type PropertyValueMap } from "lit";
-import { property, state } from "lit/decorators.js";
-import { isContextMixin } from "../gui/ContextMixin.js";
-import type { ControllableInterface } from "../gui/Controllable.js";
-import { efContext } from "../gui/efContext.js";
-import { withSpan } from "../otel/tracingHelpers.js";
-import type { AudioSpan } from "../transcoding/types/index.ts";
-import { UrlGenerator } from "../transcoding/utils/UrlGenerator.ts";
-import { makeAudioBufferTask } from "./EFMedia/audioTasks/makeAudioBufferTask.ts";
-import { makeAudioFrequencyAnalysisTask } from "./EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.ts";
-import { makeAudioInitSegmentFetchTask } from "./EFMedia/audioTasks/makeAudioInitSegmentFetchTask.ts";
-import { makeAudioInputTask } from "./EFMedia/audioTasks/makeAudioInputTask.ts";
-import { makeAudioSeekTask } from "./EFMedia/audioTasks/makeAudioSeekTask.ts";
-import { makeAudioSegmentFetchTask } from "./EFMedia/audioTasks/makeAudioSegmentFetchTask.ts";
-import { makeAudioSegmentIdTask } from "./EFMedia/audioTasks/makeAudioSegmentIdTask.ts";
-import { makeAudioTimeDomainAnalysisTask } from "./EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.ts";
-import { fetchAudioSpanningTime } from "./EFMedia/shared/AudioSpanUtils.ts";
-import { makeMediaEngineTask } from "./EFMedia/tasks/makeMediaEngineTask.ts";
-import { EFSourceMixin } from "./EFSourceMixin.js";
-import { EFTemporal } from "./EFTemporal.js";
-import { FetchMixin } from "./FetchMixin.js";
-import { renderTemporalAudio } from "./renderTemporalAudio.js";
-import { EFTargetable } from "./TargetController.ts";
-
-// EF_FRAMEGEN is a global instance created in EF_FRAMEGEN.ts
-declare global {
-  var EF_FRAMEGEN: import("../EF_FRAMEGEN.js").EFFramegen;
-}
-
-const freqWeightsCache = new Map<number, Float32Array>();
-
-export class IgnorableError extends Error {}
-
-export const deepGetMediaElements = (
-  element: Element,
-  medias: EFMedia[] = [],
-) => {
-  for (const child of Array.from(element.children)) {
-    if (child instanceof EFMedia) {
-      medias.push(child);
-    } else {
-      deepGetMediaElements(child, medias);
-    }
-  }
-  return medias;
-};
-
-export class EFMedia extends EFTargetable(
-  EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
-    assetType: "isobmff_files",
-  }),
-) {
-  @provide({ context: efContext })
-  get efContext(): ControllableInterface | null {
-    return this.rootTimegroup ?? this;
-  }
-
-  // Sample buffer size configuration
-  static readonly VIDEO_SAMPLE_BUFFER_SIZE = 30;
-  static readonly AUDIO_SAMPLE_BUFFER_SIZE = 120;
-
-  static get observedAttributes() {
-    // biome-ignore lint/complexity/noThisInStatic: We need to access super
-    const parentAttributes = super.observedAttributes || [];
-    return [
-      ...parentAttributes,
-      "mute",
-      "fft-size",
-      "fft-decay",
-      "fft-gain",
-      "interpolate-frequencies",
-      "asset-id",
-      "audio-buffer-duration",
-      "max-audio-buffer-fetches",
-      "enable-audio-buffering",
-      "sourcein",
-      "sourceout",
-    ];
-  }
-
-  static styles = [
-    css`
-      :host {
-        display: block;
-        position: relative;
-        overflow: hidden;
-      }
-    `,
-  ];
-
-  /**
-   * Duration in milliseconds for audio buffering ahead of current time
-   * @domAttribute "audio-buffer-duration"
-   */
-  @property({ type: Number, attribute: "audio-buffer-duration" })
-  audioBufferDurationMs = 10000; // 10 seconds - reasonable for JIT encoding
-
-  /**
-   * Maximum number of concurrent audio segment fetches for buffering
-   * @domAttribute "max-audio-buffer-fetches"
-   */
-  @property({ type: Number, attribute: "max-audio-buffer-fetches" })
-  maxAudioBufferFetches = 2;
-
-  /**
-   * Enable/disable audio buffering system
-   * @domAttribute "enable-audio-buffering"
-   */
-  @property({ type: Boolean, attribute: "enable-audio-buffering" })
-  enableAudioBuffering = true;
-
-  /**
-   * Mute/unmute the media element
-   * @domAttribute "mute"
-   */
-  @property({
-    type: Boolean,
-    attribute: "mute",
-    reflect: true,
-  })
-  mute = false;
-
-  /**
-   * FFT size for frequency analysis
-   * @domAttribute "fft-size"
-   */
-  @property({ type: Number, attribute: "fft-size", reflect: true })
-  fftSize = 128;
-
-  /**
-   * FFT decay rate for frequency analysis
-   * @domAttribute "fft-decay"
-   */
-  @property({ type: Number, attribute: "fft-decay", reflect: true })
-  fftDecay = 8;
-
-  /**
-   * FFT gain for frequency analysis
-   * @domAttribute "fft-gain"
-   */
-  @property({ type: Number, attribute: "fft-gain", reflect: true })
-  fftGain = 3.0;
-
-  /**
-   * Enable/disable frequency interpolation
-   * @domAttribute "interpolate-frequencies"
-   */
-  @property({
-    type: Boolean,
-    attribute: "interpolate-frequencies",
-    reflect: true,
-  })
-  interpolateFrequencies = false;
-
-  // Update FREQ_WEIGHTS to use the instance fftSize instead of a static value
-  get FREQ_WEIGHTS() {
-    if (freqWeightsCache.has(this.fftSize)) {
-      // biome-ignore lint/style/noNonNullAssertion: We know the value is set due to the guard above
-      return freqWeightsCache.get(this.fftSize)!;
-    }
-
-    const weights = new Float32Array(this.fftSize / 2).map((_, i) => {
-      const frequency = (i * 48000) / this.fftSize;
-      if (frequency < 60) return 0.3;
-      if (frequency < 250) return 0.4;
-      if (frequency < 500) return 0.6;
-      if (frequency < 2000) return 0.8;
-      if (frequency < 4000) return 1.2;
-      if (frequency < 8000) return 1.6;
-      return 2.0;
-    });
-
-    freqWeightsCache.set(this.fftSize, weights);
-    return weights;
-  }
-
-  // Helper getter for backwards compatibility
-  get shouldInterpolateFrequencies() {
-    return this.interpolateFrequencies;
-  }
-
-  get urlGenerator() {
-    return new UrlGenerator(() => this.apiHost ?? "");
-  }
-
-  mediaEngineTask = makeMediaEngineTask(this);
-
-  audioSegmentIdTask = makeAudioSegmentIdTask(this);
-  audioInitSegmentFetchTask = makeAudioInitSegmentFetchTask(this);
-  audioSegmentFetchTask = makeAudioSegmentFetchTask(this);
-  audioInputTask = makeAudioInputTask(this);
-  audioSeekTask = makeAudioSeekTask(this);
-
-  audioBufferTask = makeAudioBufferTask(this);
-
-  // Audio analysis tasks for frequency and time domain analysis
-  byteTimeDomainTask = makeAudioTimeDomainAnalysisTask(this);
-  frequencyDataTask = makeAudioFrequencyAnalysisTask(this);
-
-  /**
-   * The unique identifier for the media asset.
-   * This property can be set programmatically or via the "asset-id" attribute.
-   * @domAttribute "asset-id"
-   */
-  @property({ type: String, attribute: "asset-id", reflect: true })
-  assetId: string | null = null;
-
-  get intrinsicDurationMs() {
-    return this.mediaEngineTask.value?.durationMs ?? 0;
-  }
-
-  protected updated(
-    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
-  ): void {
-    super.updated(changedProperties);
-
-    // Check if our timeline position has actually changed, even if ownCurrentTimeMs isn't tracked as a property
-    const newCurrentSourceTimeMs = this.currentSourceTimeMs;
-    if (newCurrentSourceTimeMs !== this.desiredSeekTimeMs) {
-      this.executeSeek(newCurrentSourceTimeMs);
-    }
-
-    if (changedProperties.has("ownCurrentTimeMs")) {
-      this.executeSeek(this.currentSourceTimeMs);
-    }
-
-    // Check if trim/source properties changed that affect duration
-    const durationAffectingProps = [
-      "_trimStartMs",
-      "_trimEndMs",
-      "_sourceInMs",
-      "_sourceOutMs",
-    ];
-
-    const hasDurationChange = durationAffectingProps.some((prop) =>
-      changedProperties.has(prop),
-    );
-
-    if (hasDurationChange) {
-      // Notify parent timegroup to recalculate its duration (same pattern as EFCaptions)
-      if (this.parentTimegroup) {
-        this.parentTimegroup.requestUpdate("durationMs");
-        this.parentTimegroup.requestUpdate("currentTime");
-
-        // Also find and directly notify any context provider (ContextMixin)
-        let parent = this.parentNode;
-        while (parent) {
-          if (isContextMixin(parent)) {
-            parent.dispatchEvent(
-              new CustomEvent("child-duration-changed", {
-                detail: { source: this },
-              }),
-            );
-            break;
-          }
-          parent = parent.parentNode;
-        }
-      }
-    }
-  }
-
-  get hasOwnDuration() {
-    return true;
-  }
-
-  @state()
-  private _desiredSeekTimeMs = 0; // Initialize to 0 for proper segment loading
-
-  get desiredSeekTimeMs() {
-    return this._desiredSeekTimeMs;
-  }
-
-  set desiredSeekTimeMs(value: number) {
-    if (this._desiredSeekTimeMs !== value) {
-      this._desiredSeekTimeMs = value;
-    }
-  }
-
-  protected async executeSeek(seekToMs: number) {
-    // The seekToMs parameter should be the timeline-relative media time
-    // calculated from currentSourceTimeMs which includes timeline positioning
-    this.desiredSeekTimeMs = seekToMs;
-  }
-
-  /**
-   * Main integration method for EFTimegroup audio playback
-   * Now powered by clean, testable utility functions
-   * Returns undefined if no audio rendition is available
-   */
-  async fetchAudioSpanningTime(
-    fromMs: number,
-    toMs: number,
-    signal: AbortSignal = new AbortController().signal,
-  ): Promise<AudioSpan | undefined> {
-    return withSpan(
-      "media.fetchAudioSpanningTime",
-      {
-        elementId: this.id || "unknown",
-        tagName: this.tagName.toLowerCase(),
-        fromMs,
-        toMs,
-        durationMs: toMs - fromMs,
-        src: this.src || "none",
-      },
-      undefined,
-      async () => {
-        return fetchAudioSpanningTime(this, fromMs, toMs, signal);
-      },
-    );
-  }
-
-  /**
-   * Wait for media engine to load and determine duration
-   * Ensures media is ready for playback
-   */
-  async waitForMediaDurations(): Promise<void> {
-    if (this.mediaEngineTask.value) {
-      return;
-    }
-    await this.mediaEngineTask.run();
-  }
-
-  /**
-   * Returns media elements for playback audio rendering
-   * For standalone media, returns [this]; for timegroups, returns all descendants
-   * Used by PlaybackController for audio-driven playback
-   */
-  getMediaElements(): EFMedia[] {
-    return [this];
-  }
-
-  /**
-   * Render audio buffer for playback
-   * Called by PlaybackController during live playback
-   * Delegates to shared renderTemporalAudio utility for consistent behavior
-   */
-  async renderAudio(fromMs: number, toMs: number): Promise<AudioBuffer> {
-    return renderTemporalAudio(this, fromMs, toMs);
-  }
-}

package/src/elements/EFSourceMixin.ts

@@ -1,60 +0,0 @@
-import { Task } from "@lit/task";
-import type { LitElement } from "lit";
-import { property } from "lit/decorators/property.js";
-
-export declare class EFSourceMixinInterface {
-  apiHost?: string;
-  productionSrc(): string;
-  src: string;
-}
-
-interface EFSourceMixinOptions {
-  assetType: string;
-}
-type Constructor<T = {}> = new (...args: any[]) => T;
-export function EFSourceMixin<T extends Constructor<LitElement>>(
-  superClass: T,
-  options: EFSourceMixinOptions,
-) {
-  class EFSourceElement extends superClass {
-    get apiHost() {
-      const apiHost =
-        this.closest("ef-configuration")?.apiHost ??
-        this.closest("ef-workbench")?.apiHost ??
-        this.closest("ef-preview")?.apiHost;
-
-      return apiHost || "https://editframe.dev";
-    }
-
-    @property({ type: String })
-    src = "";
-
-    productionSrc() {
-      if (!this.md5SumLoader.value) {
-        throw new Error(
-          `MD5 sum not available for ${this}. Cannot generate production URL`,
-        );
-      }
-
-      if (!this.apiHost) {
-        throw new Error(
-          `apiHost not available for ${this}. Cannot generate production URL`,
-        );
-      }
-
-      return `${this.apiHost}/api/v1/${options.assetType}/${this.md5SumLoader.value}`;
-    }
-
-    md5SumLoader = new Task(this, {
-      autoRun: false,
-      args: () => [this.src] as const,
-      task: async ([src], { signal }) => {
-        const md5Path = `/@ef-asset/${src}`;
-        const response = await fetch(md5Path, { method: "HEAD", signal });
-        return response.headers.get("etag") ?? undefined;
-      },
-    });
-  }
-
-  return EFSourceElement as Constructor<EFSourceMixinInterface> & T;
-}

package/src/elements/EFSurface.browsertest.ts

@@ -1,151 +0,0 @@
-import { html, render } from "lit";
-import { beforeEach, describe } from "vitest";
-
-import { test as baseTest } from "../../test/useMSW.js";
-
-import "./EFVideo.js";
-import "./EFTimegroup.js";
-import "../gui/EFPreview.js";
-import "../gui/EFWorkbench.js";
-import "./EFSurface.js";
-
-import type { EFSurface } from "./EFSurface.js";
-import type { EFTimegroup } from "./EFTimegroup.js";
-import type { EFVideo } from "./EFVideo.js";
-
-beforeEach(() => {
-  localStorage.clear();
-});
-
-const surfaceTest = baseTest.extend<{
-  timegroup: EFTimegroup;
-  video: EFVideo;
-  surface: EFSurface;
-}>({
-  timegroup: async ({}, use) => {
-    const container = document.createElement("div");
-    render(
-      html`
-        <ef-configuration api-host="http://localhost:63315" signing-url="">
-          <ef-preview>
-            <ef-timegroup id="tg" mode="sequence" class="relative h-[360px] w-[640px] overflow-hidden bg-black">
-              <ef-video id="vid" src="bars-n-tone.mp4" style="width: 100%; height: 100%;"></ef-video>
-              <ef-surface id="surf" target="vid" style="position: absolute; inset: 0;"></ef-surface>
-            </ef-timegroup>
-          </ef-preview>
-        </ef-configuration>
-      `,
-      container,
-    );
-    document.body.appendChild(container);
-    const tg = container.querySelector("#tg") as EFTimegroup;
-    await tg.updateComplete;
-    await use(tg);
-    container.remove();
-  },
-  video: async ({ timegroup }, use) => {
-    const video = timegroup.querySelector("#vid") as EFVideo;
-    await video.updateComplete;
-    await use(video);
-  },
-  surface: async ({ timegroup }, use) => {
-    const surface = timegroup.querySelector("#surf") as unknown as EFSurface;
-    await surface.updateComplete;
-    await use(surface);
-  },
-});
-
-describe("EFSurface", () => {
-  surfaceTest("defines and renders a canvas", async ({ expect }) => {
-    const el = document.createElement("ef-surface");
-    document.body.appendChild(el);
-    await (el as any).updateComplete;
-    const canvas = el.shadowRoot?.querySelector("canvas");
-    expect(canvas).toBeTruthy();
-    expect((canvas as HTMLCanvasElement).tagName).toBe("CANVAS");
-    el.remove();
-  });
-
-  surfaceTest(
-    "mirrors video canvas after a seek via EFTimegroup",
-    async ({ timegroup, video, surface, expect }) => {
-      // Ensure media engine initialized
-      await video.mediaEngineTask.run();
-
-      // Seek to a known time through timegroup (triggers frame tasks)
-      timegroup.currentTimeMs = 3000;
-      await timegroup.seekTask.taskComplete;
-
-      // After scheduling, surface should have mirrored pixel dimensions
-      const videoCanvas = (video as any).canvasElement as
-        | HTMLCanvasElement
-        | undefined;
-      const surfaceCanvas =
-        (surface.shadowRoot?.querySelector("canvas") as HTMLCanvasElement) ??
-        undefined;
-
-      expect(videoCanvas).toBeTruthy();
-      expect(surfaceCanvas).toBeTruthy();
-      expect(videoCanvas!.width).toBeGreaterThan(0);
-      expect(videoCanvas!.height).toBeGreaterThan(0);
-
-      // Surface copies pixel dimensions
-      expect(surfaceCanvas!.width).toBe(videoCanvas!.width);
-      expect(surfaceCanvas!.height).toBe(videoCanvas!.height);
-    },
-  );
-
-  surfaceTest(
-    "supports multiple surfaces mirroring the same source",
-    async ({ expect }) => {
-      const container = document.createElement("div");
-      render(
-        html`
-        <ef-configuration api-host="http://localhost:63315" signing-url="">
-          <ef-preview>
-            <ef-timegroup mode="sequence" class="relative h-[360px] w-[640px] overflow-hidden bg-black">
-              <ef-video id="v" src="bars-n-tone.mp4" style="width: 100%; height: 100%;"></ef-video>
-              <ef-surface id="s1" target="v" style="position: absolute; inset: 0;"></ef-surface>
-              <ef-surface id="s2" target="v" style="position: absolute; inset: 0;"></ef-surface>
-            </ef-timegroup>
-          </ef-preview>
-        </ef-configuration>
-      `,
-        container,
-      );
-      document.body.appendChild(container);
-      const timegroup = container.querySelector("ef-timegroup") as EFTimegroup;
-      const video = container.querySelector("ef-video") as EFVideo;
-      const s1 = container.querySelector("#s1") as unknown as EFSurface;
-      const s2 = container.querySelector("#s2") as unknown as EFSurface;
-      await timegroup.updateComplete;
-      await video.mediaEngineTask.run();
-
-      timegroup.currentTimeMs = 1000;
-      await timegroup.seekTask.taskComplete;
-
-      const vCanvas = (video as any).canvasElement as HTMLCanvasElement;
-      const c1 = s1.shadowRoot!.querySelector("canvas") as HTMLCanvasElement;
-      const c2 = s2.shadowRoot!.querySelector("canvas") as HTMLCanvasElement;
-
-      expect(vCanvas.width).toBeGreaterThan(0);
-      expect(c1.width).toBe(vCanvas.width);
-      expect(c2.width).toBe(vCanvas.width);
-      expect(c1.height).toBe(vCanvas.height);
-      expect(c2.height).toBe(vCanvas.height);
-
-      container.remove();
-    },
-  );
-
-  surfaceTest(
-    "handles missing video gracefully (no throw)",
-    async ({ expect }) => {
-      const el = document.createElement("ef-surface") as any;
-      document.body.appendChild(el);
-      await el.updateComplete;
-      await expect(el.frameTask.run()).resolves.toBeUndefined();
-      el.remove();
-    },
-  );
-});

package/src/elements/EFSurface.ts

@@ -1,142 +0,0 @@
-import { Task } from "@lit/task";
-import { css, html, LitElement } from "lit";
-import { customElement, property, state } from "lit/decorators.js";
-import { createRef, ref } from "lit/directives/ref.js";
-import type { ContextMixinInterface } from "../gui/ContextMixin.ts";
-import { TargetController } from "./TargetController.ts";
-
-@customElement("ef-surface")
-export class EFSurface extends LitElement {
-  static styles = [
-    css`
-      :host {
-        display: block;
-        position: relative;
-      }
-      canvas {
-        all: inherit;
-        width: 100%;
-        height: 100%;
-        display: block;
-      }
-    `,
-  ];
-
-  canvasRef = createRef<HTMLCanvasElement>();
-
-  // @ts-expect-error controller is intentionally not referenced directly
-  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: Used for side effects
-  #targetController: TargetController = new TargetController(this);
-
-  @state()
-  targetElement: ContextMixinInterface | null = null;
-
-  @property({ type: String })
-  target = "";
-
-  render() {
-    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
-  }
-
-  // Provide minimal temporal-like properties so EFTimegroup can schedule us
-  get rootTimegroup(): any {
-    // Prefer the target element's root timegroup if available
-    const target: any = this.targetElement;
-    if (target && "rootTimegroup" in target) {
-      return target.rootTimegroup;
-    }
-    // Fallback: nearest containing timegroup if any
-    let root: any = this.closest("ef-timegroup");
-    while (root?.parentTimegroup) {
-      root = root.parentTimegroup;
-    }
-    return root;
-  }
-
-  get currentTimeMs(): number {
-    return this.rootTimegroup?.currentTimeMs ?? 0;
-  }
-
-  get durationMs(): number {
-    return this.rootTimegroup?.durationMs ?? 0;
-  }
-
-  get startTimeMs(): number {
-    return this.rootTimegroup?.startTimeMs ?? 0;
-  }
-
-  get endTimeMs(): number {
-    return this.startTimeMs + this.durationMs;
-  }
-
-  /**
-   * Minimal integration with EFTimegroup's frame scheduling:
-   * - Waits for the target video element's frameTask to complete (ensuring it painted)
-   * - Copies the target's canvas into this element's canvas
-   */
-  frameTask = new Task(this, {
-    autoRun: false,
-    args: () => [this.targetElement] as const,
-    task: async ([target]) => {
-      if (!target) return;
-
-      // Ensure the target has painted its frame for this tick
-      try {
-        const maybeTask = (target as any).frameTask;
-        if (maybeTask && typeof maybeTask.run === "function") {
-          // Run (idempotent) and then wait for completion
-          maybeTask.run();
-          await maybeTask.taskComplete;
-        }
-      } catch (_err) {
-        // Best-effort; continue to attempt copy
-      }
-
-      this.copyFromTarget(target);
-    },
-  });
-
-  protected updated(): void {
-    if (this.targetElement) {
-      this.copyFromTarget(this.targetElement);
-    }
-  }
-
-  // Target resolution is handled by TargetController. No implicit discovery.
-
-  private getSourceCanvas(from: Element): HTMLCanvasElement | null {
-    const anyEl = from as any;
-    if ("canvasElement" in anyEl) {
-      return anyEl.canvasElement ?? null;
-    }
-    const sr = (from as HTMLElement).shadowRoot;
-    if (sr) {
-      const c = sr.querySelector("canvas");
-      return (c as HTMLCanvasElement) ?? null;
-    }
-    return null;
-  }
-
-  private copyFromTarget(target: Element) {
-    const dst = this.canvasRef.value;
-    const src = this.getSourceCanvas(target);
-    if (!dst || !src) return;
-    if (!src.width || !src.height) return;
-
-    // Match source pixel size for a faithful mirror; layout scaling is handled by CSS
-    if (dst.width !== src.width || dst.height !== src.height) {
-      dst.width = src.width;
-      dst.height = src.height;
-    }
-
-    const ctx = dst.getContext("2d");
-    if (!ctx) return;
-    ctx.drawImage(src, 0, 0, dst.width, dst.height);
-  }
-}
-
-declare global {
-  interface HTMLElementTagNameMap {
-    "ef-surface": EFSurface;
-  }
-}