@editframe/elements 0.21.0-beta.0 → 0.23.6-beta.0

package/src/elements/EFTimegroup.ts

@@ -5,11 +5,12 @@ import { css, html, LitElement, type PropertyValues } from "lit";
 import { customElement, property } from "lit/decorators.js";
 
 import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
+import { EF_RENDERING } from "../EF_RENDERING.js";
 import { isContextMixin } from "../gui/ContextMixin.js";
+import { efContext } from "../gui/efContext.js";
+import { TWMixin } from "../gui/TWMixin.js";
 import { isTracingEnabled, withSpan } from "../otel/tracingHelpers.js";
-import type { AudioSpan } from "../transcoding/types/index.ts";
-import { durationConverter } from "./durationConverter.js";
-import { deepGetMediaElements } from "./EFMedia.js";
+import { deepGetMediaElements, type EFMedia } from "./EFMedia.js";
 import {
   deepGetElementsWithFrameTasks,
   EFTemporal,
@@ -18,12 +19,19 @@ import {
   shallowGetTemporalElements,
   timegroupContext,
 } from "./EFTemporal.js";
+import { parseTimeToMs } from "./parseTimeToMs.js";
+import { renderTemporalAudio } from "./renderTemporalAudio.js";
+import { EFTargetable } from "./TargetController.js";
 import { TimegroupController } from "./TimegroupController.js";
 import {
   evaluateTemporalStateForAnimation,
   updateAnimations,
 } from "./updateAnimations.ts";
 
+declare global {
+  var EF_DEV_WORKBENCH: boolean | undefined;
+}
+
 const log = debug("ef:elements:EFTimegroup");
 
 // Cache for sequence mode duration calculations to avoid O(n) recalculation
@@ -48,13 +56,24 @@ export const shallowGetTimegroups = (
 };
 
 @customElement("ef-timegroup")
-export class EFTimegroup extends EFTemporal(LitElement) {
+export class EFTimegroup extends EFTargetable(EFTemporal(TWMixin(LitElement))) {
+  static get observedAttributes(): string[] {
+    // biome-ignore lint/complexity/noThisInStatic: It's okay to use this here
+    const parentAttributes = super.observedAttributes || [];
+    return [...parentAttributes, "mode", "overlap", "currenttime", "fit"];
+  }
+
   static styles = css`
     :host {
       display: block;
+      position: relative;
+      overflow: hidden;
+    }
+
+    ::slotted(ef-timegroup) {
+      position: absolute;
       width: 100%;
       height: 100%;
-      position: absolute;
       top: 0;
       left: 0;
     }
@@ -63,95 +82,51 @@ export class EFTimegroup extends EFTemporal(LitElement) {
   @provide({ context: timegroupContext })
   _timeGroupContext = this;
 
-  #currentTime: number | undefined = undefined;
-
-  @property({
-    type: String,
-    attribute: "mode",
-  })
-  set mode(value: "fit" | "fixed" | "sequence" | "contain") {
-    // Invalidate duration cache when mode changes
-    sequenceDurationCache.delete(this);
-    this._mode = value;
-  }
-
-  get mode() {
-    return this._mode;
-  }
-
-  private _mode: "fit" | "fixed" | "sequence" | "contain" = "contain";
+  @provide({ context: efContext })
+  efContext = this;
 
-  @property({
-    type: Number,
-    converter: durationConverter,
-    attribute: "overlap",
-  })
-  set overlapMs(value: number) {
-    // Invalidate duration cache when overlap changes
-    sequenceDurationCache.delete(this);
-    this._overlapMs = value;
-  }
+  mode: "fit" | "fixed" | "sequence" | "contain" = "contain";
+  overlapMs = 0;
 
-  get overlapMs() {
-    return this._overlapMs;
+  attributeChangedCallback(
+    name: string,
+    old: string | null,
+    value: string | null,
+  ): void {
+    if (name === "mode" && value) {
+      this.mode = value as typeof this.mode;
+    }
+    if (name === "overlap" && value) {
+      this.overlapMs = parseTimeToMs(value);
+    }
+    super.attributeChangedCallback(name, old, value);
   }
 
-  private _overlapMs = 0;
-
   @property({ type: String })
   fit: "none" | "contain" | "cover" = "none";
 
   #resizeObserver?: ResizeObserver;
 
+  #currentTime: number | undefined = undefined;
   #seekInProgress = false;
-
   #pendingSeekTime: number | undefined;
-
   #processingPendingSeek = false;
 
-  #frameTaskInProgress = false;
-
-  #pendingFrameTaskRun = false;
-
-  #processingPendingFrameTask = false;
-
-  /**
-   * Throttles frameTask execution to ensure only one runs at a time while preserving the last request
-   */
   private async runThrottledFrameTask(): Promise<void> {
-    if (this.#frameTaskInProgress) {
-      this.#pendingFrameTaskRun = true;
-      // Wait for the current frame task to complete
-      while (this.#frameTaskInProgress) {
-        await this.frameTask.taskComplete;
-      }
-      return;
-    }
-
-    this.#frameTaskInProgress = true;
-
-    try {
-      await this.frameTask.run();
-    } finally {
-      this.#frameTaskInProgress = false;
-
-      if (this.#pendingFrameTaskRun && !this.#processingPendingFrameTask) {
-        this.#pendingFrameTaskRun = false;
-        this.#processingPendingFrameTask = true;
-        try {
-          await this.runThrottledFrameTask();
-        } finally {
-          this.#processingPendingFrameTask = false;
-        }
-      } else {
-        this.#pendingFrameTaskRun = false;
-      }
+    if (this.playbackController) {
+      return this.playbackController.runThrottledFrameTask();
     }
+    await this.frameTask.run();
   }
 
   @property({ type: Number, attribute: "currenttime" })
   set currentTime(time: number) {
-    time = Math.max(0, time);
+    if (this.playbackController) {
+      this.playbackController.currentTime = time;
+      return;
+    }
+
+    time = Math.max(0, Math.min(this.durationMs / 1000, time));
     if (!this.isRootTimegroup) {
       return;
     }
@@ -194,6 +169,9 @@ export class EFTimegroup extends EFTemporal(LitElement) {
   }
 
   get currentTime() {
+    if (this.playbackController) {
+      return this.playbackController.currentTime;
+    }
     return this.#currentTime ?? 0;
   }
 
@@ -205,6 +183,49 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     return this.currentTime * 1000;
   }
 
+  /**
+   * Seek to a specific time and wait for all frames to be ready.
+   * This is the recommended way to seek in tests and programmatic control.
+   *
+   * @param timeMs - Time in milliseconds to seek to
+   * @returns Promise that resolves when the seek is complete and all visible children are ready
+   */
+  async seek(timeMs: number): Promise<void> {
+    this.currentTimeMs = timeMs;
+    await this.seekTask.taskComplete;
+
+    // Handle localStorage when playbackController delegates seek
+    if (this.playbackController) {
+      this.saveTimeToLocalStorage(this.currentTime);
+    }
+
+    await this.frameTask.taskComplete;
+
+    // Ensure all visible elements have completed their reactive update cycles AND frame rendering
+    // waitForFrameTasks() calls frameTask.run() on children, but this may happen before child
+    // elements have processed property changes from requestUpdate(). To ensure frame data is
+    // accurate, we wait for updateComplete first, then ensure the frameTask has run with the
+    // updated properties. Elements like EFVideo provide waitForFrameReady() for this pattern.
+    const temporalElements = deepGetElementsWithFrameTasks(this);
+    const visibleElements = temporalElements.filter((element) => {
+      const animationState = evaluateTemporalStateForAnimation(element);
+      return animationState.isVisible;
+    });
+
+    await Promise.all(
+      visibleElements.map(async (element) => {
+        if (
+          "waitForFrameReady" in element &&
+          typeof element.waitForFrameReady === "function"
+        ) {
+          await (element as any).waitForFrameReady();
+        } else {
+          await element.updateComplete;
+        }
+      }),
+    );
+  }
+
   /**
    * Determines if this is a root timegroup (no parent timegroups)
    */
@@ -212,10 +233,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     return !this.parentTimegroup;
   }
 
-  /**
-   * Saves time to localStorage (extracted for reuse)
-   */
-  #saveTimeToLocalStorage(time: number) {
+  saveTimeToLocalStorage(time: number) {
     try {
       if (this.id && this.isConnected && !Number.isNaN(time)) {
         localStorage.setItem(this.storageKey, time.toString());
@@ -239,7 +257,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     this.requestUpdate();
   };
 
-  maybeLoadTimeFromLocalStorage() {
+  loadTimeFromLocalStorage(): number | undefined {
     if (this.id) {
       try {
         const storedValue = localStorage.getItem(this.storageKey);
@@ -251,21 +269,25 @@ export class EFTimegroup extends EFTemporal(LitElement) {
         log("Failed to load time from localStorage", error);
       }
     }
+    return undefined;
   }
 
   connectedCallback() {
     super.connectedCallback();
-    this.waitForMediaDurations().then(() => {
-      if (this.id) {
-        const maybeLoadedTime = this.maybeLoadTimeFromLocalStorage();
-        if (maybeLoadedTime !== undefined) {
-          this.currentTime = maybeLoadedTime;
+
+    if (!this.playbackController) {
+      this.waitForMediaDurations().then(() => {
+        if (this.id) {
+          const maybeLoadedTime = this.loadTimeFromLocalStorage();
+          if (maybeLoadedTime !== undefined) {
+            this.currentTime = maybeLoadedTime;
+          }
         }
-      }
-      if (EF_INTERACTIVE && this.seekTask.status === TaskStatus.INITIAL) {
-        this.seekTask.run();
-      }
-    });
+        if (EF_INTERACTIVE && this.seekTask.status === TaskStatus.INITIAL) {
+          this.seekTask.run();
+        }
+      });
+    }
 
     if (this.parentTimegroup) {
       new TimegroupController(this.parentTimegroup, this);
@@ -278,7 +300,13 @@ export class EFTimegroup extends EFTemporal(LitElement) {
 
   #previousDurationMs = 0;
 
-  protected updated(_changedProperties: PropertyValues): void {
+  protected updated(changedProperties: PropertyValues): void {
+    super.updated(changedProperties);
+
+    if (changedProperties.has("mode") || changedProperties.has("overlapMs")) {
+      sequenceDurationCache.delete(this);
+    }
+
     if (this.#previousDurationMs !== this.durationMs) {
       this.#previousDurationMs = this.durationMs;
       this.runThrottledFrameTask();
@@ -549,13 +577,31 @@ export class EFTimegroup extends EFTemporal(LitElement) {
   /**
    * Returns true if the timegroup should be wrapped with a workbench.
    *
-   * A timegroup should be wrapped with a workbench if it is the root-most timegroup
-   * and EF_INTERACTIVE is true.
+   * A timegroup should be wrapped with a workbench if:
+   * - It's being rendered (EF_RENDERING), OR
+   * - It's in interactive mode (EF_INTERACTIVE) with the dev workbench flag set
    *
-   * If the timegroup is already wrappedin a context provider like ef-preview,
+   * If the timegroup is already wrapped in a context provider like ef-preview,
    * it should NOT be wrapped in a workbench.
    */
   shouldWrapWithWorkbench() {
+    const isRendering = EF_RENDERING?.() === true;
+
+    // During rendering, always wrap with workbench (needed by EF_FRAMEGEN)
+    if (isRendering) {
+      return (
+        this.closest("ef-timegroup") === this &&
+        this.closest("ef-preview") === null &&
+        this.closest("ef-workbench") === null &&
+        this.closest("test-context") === null
+      );
+    }
+
+    // During interactive mode, respect the dev workbench flag
+    if (!globalThis.EF_DEV_WORKBENCH) {
+      return false;
+    }
+
     return (
       EF_INTERACTIVE &&
       this.closest("ef-timegroup") === this &&
@@ -588,155 +634,22 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     );
   }
 
-  async #addAudioToContext(
-    audioContext: AudioContext | OfflineAudioContext,
-    fromMs: number,
-    toMs: number,
-  ) {
-    await this.waitForMediaDurations();
-
-    // Create AbortController for audio fetch operations
-    const abortController = new AbortController();
-
-    await Promise.all(
-      deepGetMediaElements(this).map(async (mediaElement) => {
-        // Skip muted elements entirely - no audio fetching or processing needed
-        if (mediaElement.mute) {
-          return;
-        }
-
-        const mediaStartsBeforeEnd = mediaElement.startTimeMs <= toMs;
-        const mediaEndsAfterStart = mediaElement.endTimeMs >= fromMs;
-        const mediaOverlaps = mediaStartsBeforeEnd && mediaEndsAfterStart;
-        if (!mediaOverlaps) {
-          return;
-        }
-
-        // Convert from root timegroup timeline to media element's local timeline
-        const mediaLocalFromMs = Math.max(0, fromMs - mediaElement.startTimeMs);
-        const mediaLocalToMs = Math.min(
-          mediaElement.endTimeMs - mediaElement.startTimeMs,
-          toMs - mediaElement.startTimeMs,
-        );
-
-        // Skip if no valid local time range
-        if (mediaLocalFromMs >= mediaLocalToMs) {
-          return;
-        }
-
-        // Convert from local timeline to source media timeline (accounting for sourcein/sourceout)
-        const sourceInMs =
-          mediaElement.sourceInMs || mediaElement.trimStartMs || 0;
-        const mediaSourceFromMs = mediaLocalFromMs + sourceInMs;
-        const mediaSourceToMs = mediaLocalToMs + sourceInMs;
-
-        let audio: AudioSpan | undefined;
-        try {
-          audio = await mediaElement.fetchAudioSpanningTime(
-            mediaSourceFromMs,
-            mediaSourceToMs,
-            abortController.signal,
-          );
-        } catch (error) {
-          if (
-            error instanceof Error &&
-            error.message.includes("No audio track available")
-          ) {
-            return;
-          }
-          throw error;
-        }
-
-        if (!audio) {
-          return;
-        }
-
-        const bufferSource = audioContext.createBufferSource();
-        bufferSource.buffer = await audioContext.decodeAudioData(
-          await audio.blob.arrayBuffer(),
-        );
-        bufferSource.connect(audioContext.destination);
-
-        // Calculate timing for placing this audio in the output context
-        const ctxStartMs = Math.max(0, mediaElement.startTimeMs - fromMs);
-
-        // Calculate offset within the fetched audio buffer
-        // audio.startMs is now in source timeline, convert back to compare properly
-        const requestedSourceFromMs = mediaSourceFromMs;
-        const actualSourceStartMs = audio.startMs;
-        const offsetInBufferMs = requestedSourceFromMs - actualSourceStartMs;
-
-        // Ensure offset is never negative (this would cause audio scheduling errors)
-        const safeOffsetMs = Math.max(0, offsetInBufferMs);
-
-        // Calculate exact duration to play from the buffer (don't exceed what we need)
-        const requestedDurationMs = mediaSourceToMs - mediaSourceFromMs;
-        const availableAudioMs = audio.endMs - audio.startMs;
-        const actualDurationMs = Math.min(
-          requestedDurationMs,
-          availableAudioMs - safeOffsetMs,
-        );
-
-        if (actualDurationMs <= 0) {
-          return; // Skip if no valid audio duration
-        }
-
-        bufferSource.start(
-          ctxStartMs / 1000, // When to start in output context (seconds)
-          safeOffsetMs / 1000, // Offset into the fetched buffer (seconds)
-          actualDurationMs / 1000, // How long to play from buffer (seconds)
-        );
-      }),
-    );
+  /**
+   * Returns media elements for playback audio rendering
+   * For standalone media, returns [this]; for timegroups, returns all descendants
+   * Used by PlaybackController for audio-driven playback
+   */
+  getMediaElements(): EFMedia[] {
+    return deepGetMediaElements(this);
   }
 
-  async renderAudio(fromMs: number, toMs: number) {
-    return withSpan(
-      "timegroup.renderAudio",
-      {
-        timegroupId: this.id || "unknown",
-        fromMs,
-        toMs,
-        durationMs: toMs - fromMs,
-      },
-      undefined,
-      async (span) => {
-        // Here we determine the number of samples we need to render rather than the duration.
-        // We cannot tolerate having more or fewer samples than fit exactlly into AAC frames.
-        const durationMs = toMs - fromMs;
-        const duration = durationMs / 1000;
-        const exactSamples = 48000 * duration;
-        const aacFrames = exactSamples / 1024;
-        const alignedFrames = Math.round(aacFrames);
-        const contextSize = alignedFrames * 1024; // AAC-aligned sample count
-
-        if (isTracingEnabled()) {
-          span.setAttribute("contextSize", contextSize);
-          span.setAttribute("alignedFrames", alignedFrames);
-        }
-
-        // Debug logging for audio duration calculations
-        if (contextSize <= 0) {
-          throw new Error(
-            `Duration must be greater than 0 when rendering audio. ${contextSize}ms`,
-          );
-        }
-
-        let audioContext: OfflineAudioContext;
-        try {
-          audioContext = new OfflineAudioContext(2, contextSize, 48000);
-        } catch (error) {
-          throw new Error(
-            `[EFTimegroup.renderAudio] Failed to create OfflineAudioContext(2, ${contextSize}, 48000) for renderAudio(${fromMs}, ${toMs}) with contextSize=${contextSize}: ${error instanceof Error ? error.message : String(error)}. This typically happens when audio parameters are invalid (e.g., contextSize <= 0).`,
-          );
-        }
-
-        await this.#addAudioToContext(audioContext, fromMs, toMs);
-        const renderedBuffer = await audioContext.startRendering();
-
-        return renderedBuffer;
-      },
-    );
+  /**
+   * Render audio buffer for playback
+   * Called by PlaybackController during live playback
+   * Delegates to shared renderTemporalAudio utility for consistent behavior
+   */
+  async renderAudio(fromMs: number, toMs: number): Promise<AudioBuffer> {
+    return renderTemporalAudio(this, fromMs, toMs);
   }
 
   /**
@@ -780,7 +693,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
 
     await Promise.all(loaderTasks);
 
-    efElements.map((el) => {
+    efElements.forEach((el) => {
       if ("productionSrc" in el && el.productionSrc instanceof Function) {
         el.setAttribute("src", el.productionSrc());
       }
@@ -815,6 +728,11 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     args: () => [this.#pendingSeekTime ?? this.#currentTime] as const,
     onComplete: () => {},
     task: async ([targetTime]) => {
+      if (this.playbackController) {
+        await this.playbackController.seekTask.taskComplete;
+        return this.currentTime;
+      }
+
       if (!this.isRootTimegroup) {
         return;
       }
@@ -836,11 +754,11 @@ export class EFTimegroup extends EFTemporal(LitElement) {
             span.setAttribute("newTime", newTime);
           }
           // Apply the clamped time back to currentTime
+
           this.#currentTime = newTime;
           this.requestUpdate("currentTime");
           await this.runThrottledFrameTask();
-          this.#saveTimeToLocalStorage(this.#currentTime);
-          // This has to be set false here so any following seeks are not treated as pending
+          this.saveTimeToLocalStorage(this.#currentTime);
           this.#seekInProgress = false;
           return newTime;
         },