@editframe/elements 0.16.8-beta.0 → 0.18.3-beta.0

package/src/elements/EFTemporal.browsertest.ts

@@ -47,7 +47,6 @@ describe("sourcein and sourceout", () => {
       const element = document.createElement("ten-seconds");
       element.sourceInMs = 5_000;
       element.sourceOutMs = 1_000;
-      console.log(element.sourceInMs, element.sourceOutMs, element.durationMs);
       expect(element.durationMs).toBe(0);
     });
   });

package/src/elements/EFTemporal.ts

@@ -1,11 +1,10 @@
 import { consume, createContext } from "@lit/context";
+import { Task } from "@lit/task";
 import type { LitElement, PropertyValueMap, ReactiveController } from "lit";
 import { property, state } from "lit/decorators.js";
-import type { EFTimegroup } from "./EFTimegroup.js";
-
-import { Task } from "@lit/task";
 import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
 import { durationConverter } from "./durationConverter.js";
+import type { EFTimegroup } from "./EFTimegroup.js";
 
 export const timegroupContext = createContext<EFTimegroup>(
   Symbol("timeGroupContext"),
@@ -292,6 +291,10 @@ const resetStartTimeMsCache = () => {
 };
 resetStartTimeMsCache();
 
+export const flushStartTimeMsCache = () => {
+  startTimeMsCache = new WeakMap();
+};
+
 export const EFTemporal = <T extends Constructor<LitElement>>(
   superClass: T,
 ) => {
@@ -536,12 +539,19 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
           const ownIndex = siblingTemorals?.indexOf(
             this as InstanceType<Constructor<TemporalMixinInterface> & T>,
           );
+          if (ownIndex === -1) {
+            return 0;
+          }
           if (ownIndex === 0) {
             startTimeMsCache.set(this, parentTimegroup.startTimeMs);
             return parentTimegroup.startTimeMs;
           }
           const previous = siblingTemorals?.[(ownIndex ?? 0) - 1];
           if (!previous) {
+            console.error("Previous temporal element not found", {
+              ownIndex,
+              siblingTemorals,
+            });
             throw new Error("Previous temporal element not found");
           }
           startTimeMsCache.set(

package/src/elements/EFTimegroup.browsertest.ts

@@ -1,8 +1,8 @@
 import {
-  LitElement,
-  type TemplateResult,
   html,
+  LitElement,
   render as litRender,
+  type TemplateResult,
 } from "lit";
 import { assert, beforeEach, describe, test } from "vitest";
 import { EFTimegroup } from "./EFTimegroup.js";
@@ -353,7 +353,7 @@ describe("setting currentTime", () => {
     document.body.appendChild(timegroup);
     assert.isNull(localStorage.getItem(timegroup.storageKey));
     timegroup.currentTime = 5_000;
-    assert.equal(localStorage.getItem(timegroup.storageKey), "5000");
+    assert.equal(localStorage.getItem(timegroup.storageKey), "10"); // Clamped to duration
     timegroup.remove();
   });
 
@@ -392,6 +392,9 @@ describe("setting currentTime", () => {
     assert.equal(a.ownCurrentTimeMs, 2_500);
     assert.equal(b.ownCurrentTimeMs, 0);
 
+    // Wait for frame update to complete before next assignment
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
     root.currentTimeMs = 7_500;
 
     assert.equal(a.ownCurrentTimeMs, 5_000);

package/src/elements/EFTimegroup.ts

@@ -1,19 +1,21 @@
 import { provide } from "@lit/context";
-import { Task } from "@lit/task";
+import { Task, TaskStatus } from "@lit/task";
 import debug from "debug";
-import { LitElement, type PropertyValueMap, css, html } from "lit";
+import { css, html, LitElement, type PropertyValueMap } from "lit";
 import { customElement, property } from "lit/decorators.js";
 
 import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
 import { isContextMixin } from "../gui/ContextMixin.js";
+import { durationConverter } from "./durationConverter.js";
 import { deepGetMediaElements } from "./EFMedia.js";
 import {
+  deepGetElementsWithFrameTasks,
   EFTemporal,
+  flushStartTimeMsCache,
   shallowGetTemporalElements,
   timegroupContext,
 } from "./EFTemporal.js";
 import { TimegroupController } from "./TimegroupController.js";
-import { durationConverter } from "./durationConverter.js";
 import { updateAnimations } from "./updateAnimations.ts";
 
 const log = debug("ef:elements:EFTimegroup");
@@ -50,6 +52,10 @@ export class EFTimegroup extends EFTemporal(LitElement) {
 
   #currentTime = 0;
 
+  // Frame update locking mechanism (only for root timegroups)
+  private isFrameUpdateInProgress = false;
+  private queuedTimeUpdate: number | null = null;
+
   @property({
     type: String,
     attribute: "mode",
@@ -70,27 +76,84 @@ export class EFTimegroup extends EFTemporal(LitElement) {
 
   @property({ type: Number, attribute: "currenttime" })
   set currentTime(time: number) {
-    this.#currentTime = Math.max(0, Math.min(time, this.durationMs / 1000));
-    try {
-      if (this.id) {
-        if (this.isConnected) {
-          localStorage.setItem(this.storageKey, time.toString());
-        }
-      }
-    } catch (error) {
-      log("Failed to save time to localStorage", error);
+    const newTime = Math.max(0, Math.min(time, this.durationMs / 1000));
+
+    // Only apply locking mechanism for root timegroups to prevent cascade overload
+    if (this.isRootTimegroup && this.isFrameUpdateInProgress) {
+      // Queue the latest time update - only keep the most recent
+      this.queuedTimeUpdate = newTime;
+      return;
+    }
+
+    if (this.isRootTimegroup) {
+      this.#executeTimeUpdate(newTime);
+    } else {
+      // Non-root timegroups update immediately (no cascade risk)
+      this.#currentTime = newTime;
+      this.#saveTimeToLocalStorage(newTime);
     }
   }
+
   get currentTime() {
     return this.#currentTime;
   }
+
   get currentTimeMs() {
     return this.currentTime * 1000;
   }
+
   set currentTimeMs(ms: number) {
     this.currentTime = ms / 1000;
   }
 
+  /**
+   * Determines if this is a root timegroup (no parent timegroups)
+   */
+  get isRootTimegroup(): boolean {
+    return this.closest("ef-timegroup") === this;
+  }
+
+  /**
+   * Executes time update with frame locking for root timegroups
+   */
+  async #executeTimeUpdate(time: number) {
+    this.isFrameUpdateInProgress = true;
+    this.#currentTime = time;
+
+    try {
+      // Save to localStorage
+      this.#saveTimeToLocalStorage(time);
+
+      // Wait for any pending frame tasks to complete before allowing next update
+      await this.waitForFrameTasks();
+    } catch (error) {
+      console.error("⚠️ [TIME_UPDATE_ERROR] Error during frame update:", error);
+    } finally {
+      this.isFrameUpdateInProgress = false;
+
+      // Process queued update if any (ensures latest scrub position is processed)
+      if (this.queuedTimeUpdate !== null && this.queuedTimeUpdate !== time) {
+        const nextTime = this.queuedTimeUpdate;
+        this.queuedTimeUpdate = null;
+        // Schedule on next tick to avoid recursive call stack
+        setTimeout(() => this.#executeTimeUpdate(nextTime), 0);
+      }
+    }
+  }
+
+  /**
+   * Saves time to localStorage (extracted for reuse)
+   */
+  #saveTimeToLocalStorage(time: number) {
+    try {
+      if (this.id && this.isConnected) {
+        localStorage.setItem(this.storageKey, time.toString());
+      }
+    } catch (error) {
+      log("Failed to save time to localStorage", error);
+    }
+  }
+
   render() {
     return html`<slot></slot> `;
   }
@@ -110,7 +173,7 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     super.connectedCallback();
     if (this.id) {
       this.waitForMediaDurations().then(() => {
-        this.#currentTime = this.maybeLoadTimeFromLocalStorage();
+        this.currentTime = this.maybeLoadTimeFromLocalStorage();
       });
     }
 
@@ -197,6 +260,30 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     }
   }
 
+  async getPendingFrameTasks() {
+    await this.updateComplete;
+    const temporals = deepGetElementsWithFrameTasks(this);
+    return temporals
+      .map((temporal) => temporal.frameTask)
+      .filter((task) => task.status < TaskStatus.COMPLETE);
+  }
+
+  async waitForFrameTasks() {
+    const limit = 10;
+    let step = 0;
+    await this.updateComplete;
+    while (step < limit) {
+      step++;
+      let pendingTasks = await this.getPendingFrameTasks();
+      await Promise.all(pendingTasks.map((task) => task.taskComplete));
+      await this.updateComplete;
+      pendingTasks = await this.getPendingFrameTasks();
+      if (pendingTasks.length === 0) {
+        break;
+      }
+    }
+  }
+
   /**
    * Wait for all media elements to load their initial segments.
    * Ideally we would only need the extracted index json data, but
@@ -204,10 +291,30 @@ export class EFTimegroup extends EFTemporal(LitElement) {
    * in calculations and it was not clear why.
    */
   async waitForMediaDurations() {
+    // We must await updateComplete to ensure all media elements inside this are connected
+    // and will match deepGetMediaElements
+    await this.updateComplete;
     const mediaElements = deepGetMediaElements(this);
-    return await Promise.all(
-      mediaElements.map((m) => m.trackFragmentIndexLoader.taskComplete),
-    );
+    // Then, we must await the fragmentIndexTask to ensure all media elements have their
+    // fragment index loaded, which is where their duration is parsed from.
+    await Promise.all(mediaElements.map((m) => m.mediaEngineTask.taskComplete));
+
+    // After waiting for durations, we must force some updates to cascade and ensure all temporal elements
+    // have correct durations and start times. It is not ideal that we have to do this inside here,
+    // but it is the best current way to ensure that all temporal elements have correct durations and start times.
+
+    // Next, we must flush the startTimeMs cache to ensure all media elements have their
+    // startTimeMs parsed fresh, otherwise the startTimeMs is cached per animation frame.
+    flushStartTimeMsCache();
+
+    // Request an update to the currentTime of this group, ensuring that time updates will cascade
+    // down to children, forcing sequence groups to arrange correctly.
+    // This also makes the filmstrip update correctly.
+    this.requestUpdate("currentTime");
+    // Finally, we must await updateComplete to ensure all temporal elements have their
+    // currentTime updated and all animations have run.
+
+    await this.updateComplete;
   }
 
   get childTemporals() {
@@ -290,16 +397,40 @@ export class EFTimegroup extends EFTemporal(LitElement) {
   ) {
     await this.waitForMediaDurations();
 
+    // Create AbortController for audio fetch operations
+    const abortController = new AbortController();
+
     await Promise.all(
       deepGetMediaElements(this).map(async (mediaElement) => {
+        // Skip muted elements entirely - no audio fetching or processing needed
+        if (mediaElement.mute) {
+          return;
+        }
+
         const mediaStartsBeforeEnd = mediaElement.startTimeMs <= toMs;
         const mediaEndsAfterStart = mediaElement.endTimeMs >= fromMs;
         const mediaOverlaps = mediaStartsBeforeEnd && mediaEndsAfterStart;
-        if (!mediaOverlaps || mediaElement.defaultAudioTrackId === undefined) {
+        if (!mediaOverlaps) {
           return;
         }
 
-        const audio = await mediaElement.fetchAudioSpanningTime(fromMs, toMs);
+        // Convert from root timegroup timeline to media element's local timeline
+        const mediaLocalFromMs = Math.max(0, fromMs - mediaElement.startTimeMs);
+        const mediaLocalToMs = Math.min(
+          mediaElement.endTimeMs - mediaElement.startTimeMs,
+          toMs - mediaElement.startTimeMs,
+        );
+
+        // Skip if no valid local time range
+        if (mediaLocalFromMs >= mediaLocalToMs) {
+          return;
+        }
+
+        const audio = await mediaElement.fetchAudioSpanningTime(
+          mediaLocalFromMs, // ✅ Now using media element's local timeline
+          mediaLocalToMs, // ✅ Now using media element's local timeline
+          abortController.signal,
+        );
         if (!audio) {
           throw new Error("Failed to fetch audio");
         }
@@ -310,16 +441,25 @@ export class EFTimegroup extends EFTemporal(LitElement) {
         );
         bufferSource.connect(audioContext.destination);
 
+        // Calculate timing for placing this audio in the output context
         const ctxStartMs = Math.max(0, mediaElement.startTimeMs - fromMs);
         const ctxEndMs = mediaElement.endTimeMs - fromMs;
         const ctxDurationMs = ctxEndMs - ctxStartMs;
 
-        const offset =
-          Math.max(0, fromMs - mediaElement.startTimeMs) - audio.startMs;
+        // Calculate offset within the fetched audio buffer
+        // Since we now use local timeline coordinates, audio.startMs is relative to media start
+        const requestedOffsetInMedia = mediaLocalFromMs; // Already in local timeline
+        const actualOffsetInBuffer = requestedOffsetInMedia - audio.startMs; // Both in local timeline
+
+        // Ensure offset is never negative (this would cause audio scheduling errors)
+        const safeOffset = Math.max(0, actualOffsetInBuffer);
+
+        if (safeOffset !== actualOffsetInBuffer) {
+        }
 
         bufferSource.start(
           ctxStartMs / 1000,
-          offset / 1000,
+          safeOffset / 1000,
           ctxDurationMs / 1000,
         );
       }),
@@ -327,14 +467,68 @@ export class EFTimegroup extends EFTemporal(LitElement) {
   }
 
   async renderAudio(fromMs: number, toMs: number) {
+    // Here we determine the number of samples we need to render rather than the duration.
+    // We cannot tolerate having more or fewer samples than fit exactlly into AAC frames.
     const durationMs = toMs - fromMs;
-    const audioContext = new OfflineAudioContext(
-      2,
-      Math.round((48000 * durationMs) / 1000),
-      48000,
-    );
+    const duration = durationMs / 1000;
+    const exactSamples = 48000 * duration;
+    const aacFrames = exactSamples / 1024;
+    const alignedFrames = Math.round(aacFrames);
+    const contextSize = alignedFrames * 1024; // AAC-aligned sample count
+
+    // Debug logging for audio duration calculations
+    if (contextSize <= 0) {
+      throw new Error(
+        `Duration must be greater than 0 when rendering audio. ${contextSize}ms`,
+      );
+    }
+
+    let audioContext: OfflineAudioContext;
+    try {
+      audioContext = new OfflineAudioContext(2, contextSize, 48000);
+    } catch (error) {
+      throw new Error(
+        `[EFTimegroup.renderAudio] Failed to create OfflineAudioContext(2, ${contextSize}, 48000) for renderAudio(${fromMs}, ${toMs}) with contextSize=${contextSize}: ${error instanceof Error ? error.message : String(error)}. This typically happens when audio parameters are invalid (e.g., contextSize <= 0).`,
+      );
+    }
+
     await this.#addAudioToContext(audioContext, fromMs, toMs);
-    return await audioContext.startRendering();
+    const renderedBuffer = await audioContext.startRendering();
+
+    return renderedBuffer;
+  }
+
+  /**
+   * TEMPORARY TEST METHOD: Renders audio and immediately plays it back
+   * Usage: timegroup.testPlayAudio(0, 5000) // Play first 5 seconds
+   */
+  async testPlayAudio(fromMs: number, toMs: number) {
+    try {
+      // Render the audio using the existing renderAudio method
+      const renderedBuffer = await this.renderAudio(fromMs, toMs);
+
+      // Create a regular AudioContext for playback
+      const playbackContext = new AudioContext();
+
+      // Create a buffer source and connect it
+      const bufferSource = playbackContext.createBufferSource();
+      bufferSource.buffer = renderedBuffer;
+      bufferSource.connect(playbackContext.destination);
+
+      // Start playback immediately
+      bufferSource.start(0);
+
+      // Return a promise that resolves when playback ends
+      return new Promise<void>((resolve) => {
+        bufferSource.onended = () => {
+          playbackContext.close();
+          resolve();
+        };
+      });
+    } catch (error) {
+      console.error("🎵 [TEST_PLAY_AUDIO] Error:", error);
+      throw error;
+    }
   }
 
   async loadMd5Sums() {