@editframe/elements 0.20.4-beta.0 → 0.23.6-beta.0

package/src/elements/EFMedia/videoTasks/makeUnifiedVideoSeekTask.ts

@@ -1,8 +1,12 @@
 import { Task } from "@lit/task";
 import type { VideoSample } from "mediabunny";
+import { withSpan } from "../../../otel/tracingHelpers.js";
 import type { VideoRendition } from "../../../transcoding/types";
+import { EFMedia } from "../../EFMedia.js";
 import type { EFVideo } from "../../EFVideo";
+import { BufferedSeekingInput } from "../BufferedSeekingInput.js";
 import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
+import { MainVideoInputCache } from "./MainVideoInputCache";
 import { ScrubInputCache } from "./ScrubInputCache";
 
 type UnifiedVideoSeekTask = Task<readonly [number], VideoSample | undefined>;
@@ -10,6 +14,9 @@ type UnifiedVideoSeekTask = Task<readonly [number], VideoSample | undefined>;
 // Shared cache for scrub inputs
 const scrubInputCache = new ScrubInputCache();
 
+// Shared cache for main video inputs
+const mainVideoInputCache = new MainVideoInputCache();
+
 export const makeUnifiedVideoSeekTask = (
   host: EFVideo,
 ): UnifiedVideoSeekTask => {
@@ -22,7 +29,7 @@ export const makeUnifiedVideoSeekTask = (
     onComplete: (_value) => {},
     task: async ([desiredSeekTimeMs], { signal }) => {
       const mediaEngine = await getLatestMediaEngine(host, signal);
-      if (!mediaEngine) return undefined;
+      if (!mediaEngine || signal.aborted) return undefined;
 
       // FIRST: Check if main quality content is already cached
       const mainRendition = mediaEngine.videoRendition;
@@ -35,13 +42,18 @@ export const makeUnifiedVideoSeekTask = (
           mainSegmentId !== undefined &&
           mediaEngine.isSegmentCached(mainSegmentId, mainRendition)
         ) {
-          // Main content is cached! Use it directly.
-          return await getMainVideoSample(
+          const result = await getMainVideoSample(
             host,
             mediaEngine,
             desiredSeekTimeMs,
             signal,
           );
+
+          if (signal.aborted) {
+            return undefined;
+          }
+
+          return result;
         }
       }
 
@@ -51,7 +63,12 @@ export const makeUnifiedVideoSeekTask = (
         desiredSeekTimeMs,
         signal,
       );
+
       if (scrubSample || signal.aborted) {
+        if (signal.aborted) {
+          return undefined;
+        }
+
         // If scrub succeeded, start background main quality upgrade (non-blocking)
         if (scrubSample) {
           startMainQualityUpgrade(
@@ -68,12 +85,18 @@ export const makeUnifiedVideoSeekTask = (
       }
 
       // THIRD: Neither are cached, fetch main video path as final fallback
-      return await getMainVideoSample(
+      const result = await getMainVideoSample(
         host,
         mediaEngine,
         desiredSeekTimeMs,
         signal,
       );
+
+      if (signal.aborted) {
+        return undefined;
+      }
+
+      return result;
     },
   });
 };
@@ -86,82 +109,124 @@ async function tryGetScrubSample(
   desiredSeekTimeMs: number,
   signal: AbortSignal,
 ): Promise<VideoSample | undefined> {
-  try {
-    // Get scrub rendition
-    let scrubRendition: VideoRendition | undefined;
-
-    // Check if media engine has a getScrubVideoRendition method (AssetMediaEngine, etc.)
-    if (typeof mediaEngine.getScrubVideoRendition === "function") {
-      scrubRendition = mediaEngine.getScrubVideoRendition();
-    } else if ("data" in mediaEngine && mediaEngine.data?.videoRenditions) {
-      // Fallback to data structure for other engines
-      scrubRendition = mediaEngine.data.videoRenditions.find(
-        (r: any) => r.id === "scrub",
-      );
-    }
+  return withSpan(
+    "video.tryGetScrubSample",
+    {
+      desiredSeekTimeMs,
+      src: mediaEngine.src || "unknown",
+    },
+    undefined,
+    async (span) => {
+      try {
+        // Get scrub rendition
+        let scrubRendition: VideoRendition | undefined;
+
+        // Check if media engine has a getScrubVideoRendition method (AssetMediaEngine, etc.)
+        if (typeof mediaEngine.getScrubVideoRendition === "function") {
+          scrubRendition = mediaEngine.getScrubVideoRendition();
+        } else if ("data" in mediaEngine && mediaEngine.data?.videoRenditions) {
+          // Fallback to data structure for other engines
+          scrubRendition = mediaEngine.data.videoRenditions.find(
+            (r: any) => r.id === "scrub",
+          );
+        }
 
-    if (!scrubRendition) return undefined;
+        if (!scrubRendition) {
+          span.setAttribute("result", "no-scrub-rendition");
+          return undefined;
+        }
 
-    const scrubRenditionWithSrc = {
-      ...scrubRendition,
-      src: mediaEngine.src,
-    };
+        const scrubRenditionWithSrc = {
+          ...scrubRendition,
+          src: mediaEngine.src,
+        };
 
-    // Check if scrub segment is cached
-    const segmentId = mediaEngine.computeSegmentId(
-      desiredSeekTimeMs,
-      scrubRenditionWithSrc,
-    );
-    if (segmentId === undefined) return undefined;
-
-    const isCached = mediaEngine.isSegmentCached(
-      segmentId,
-      scrubRenditionWithSrc,
-    );
-    if (!isCached) return undefined; // Not cached - let main video handle it
-
-    // Get cached scrub input and seek within it
-    const scrubInput = await scrubInputCache.getOrCreateInput(
-      segmentId,
-      async () => {
-        const [initSegment, mediaSegment] = await Promise.all([
-          mediaEngine.fetchInitSegment(scrubRenditionWithSrc, signal),
-          mediaEngine.fetchMediaSegment(segmentId, scrubRenditionWithSrc),
-        ]);
-
-        if (!initSegment || !mediaSegment || signal.aborted) return undefined;
-
-        const { BufferedSeekingInput } = await import(
-          "../BufferedSeekingInput.js"
+        // Check if scrub segment is cached
+        const segmentId = mediaEngine.computeSegmentId(
+          desiredSeekTimeMs,
+          scrubRenditionWithSrc,
+        );
+        if (segmentId === undefined) {
+          span.setAttribute("result", "no-segment-id");
+          return undefined;
+        }
+
+        const isCached = mediaEngine.isSegmentCached(
+          segmentId,
+          scrubRenditionWithSrc,
         );
-        const { EFMedia } = await import("../../EFMedia.js");
-
-        return new BufferedSeekingInput(
-          await new Blob([initSegment, mediaSegment]).arrayBuffer(),
-          {
-            videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
-            audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
-            startTimeOffsetMs: scrubRendition.startTimeOffsetMs,
+        span.setAttribute("isCached", isCached);
+        if (!isCached) {
+          span.setAttribute("result", "not-cached");
+          return undefined; // Not cached - let main video handle it
+        }
+
+        // Get cached scrub input and seek within it
+        const scrubInput = await scrubInputCache.getOrCreateInput(
+          segmentId,
+          async () => {
+            const [initSegment, mediaSegment] = await Promise.all([
+              mediaEngine.fetchInitSegment(scrubRenditionWithSrc, signal),
+              mediaEngine.fetchMediaSegment(segmentId, scrubRenditionWithSrc),
+            ]);
+
+            if (!initSegment || !mediaSegment || signal.aborted)
+              return undefined;
+
+            const { BufferedSeekingInput } = await import(
+              "../BufferedSeekingInput.js"
+            );
+            const { EFMedia } = await import("../../EFMedia.js");
+
+            return new BufferedSeekingInput(
+              await new Blob([initSegment, mediaSegment]).arrayBuffer(),
+              {
+                videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
+                audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
+                startTimeOffsetMs: scrubRendition.startTimeOffsetMs,
+              },
+            );
           },
         );
-      },
-    );
 
-    if (!scrubInput) return undefined;
+        if (!scrubInput) {
+          span.setAttribute("result", "no-scrub-input");
+          return undefined;
+        }
 
-    const videoTrack = await scrubInput.getFirstVideoTrack();
-    if (!videoTrack) return undefined;
+        if (signal.aborted) {
+          span.setAttribute("result", "aborted-after-scrub-input");
+          return undefined;
+        }
 
-    const sample = (await scrubInput.seek(
-      videoTrack.id,
-      desiredSeekTimeMs,
-    )) as unknown as VideoSample | undefined;
+        const videoTrack = await scrubInput.getFirstVideoTrack();
+        if (!videoTrack) {
+          span.setAttribute("result", "no-video-track");
+          return undefined;
+        }
 
-    return sample;
-  } catch (_error) {
-    if (signal.aborted) return undefined;
-    return undefined; // Scrub failed - let main video handle it
-  }
+        if (signal.aborted) {
+          span.setAttribute("result", "aborted-after-scrub-track");
+          return undefined;
+        }
+
+        const sample = (await scrubInput.seek(
+          videoTrack.id,
+          desiredSeekTimeMs,
+        )) as unknown as VideoSample | undefined;
+
+        span.setAttribute("result", sample ? "success" : "no-sample");
+        return sample;
+      } catch (_error) {
+        if (signal.aborted) {
+          span.setAttribute("result", "aborted");
+          return undefined;
+        }
+        span.setAttribute("result", "error");
+        return undefined; // Scrub failed - let main video handle it
+      }
+    },
+  );
 }
 
 /**
@@ -173,60 +238,101 @@ async function getMainVideoSample(
   desiredSeekTimeMs: number,
   signal: AbortSignal,
 ): Promise<VideoSample | undefined> {
-  try {
-    // Use existing main video task chain
-    const videoRendition = mediaEngine.getVideoRendition();
-    if (!videoRendition) {
-      throw new Error(
-        "Video rendition unavailable after checking videoRendition exists",
-      );
-    }
-
-    const segmentId = mediaEngine.computeSegmentId(
+  return withSpan(
+    "video.getMainVideoSample",
+    {
       desiredSeekTimeMs,
-      videoRendition,
-    );
-    if (segmentId === undefined) return undefined;
-
-    // Fetch main video segment
-    const [initSegment, mediaSegment] = await Promise.all([
-      mediaEngine.fetchInitSegment(videoRendition, signal),
-      mediaEngine.fetchMediaSegment(segmentId, videoRendition, signal),
-    ]);
-
-    if (!initSegment || !mediaSegment) return undefined;
-    signal.throwIfAborted();
+      src: mediaEngine.src || "unknown",
+    },
+    undefined,
+    async (span) => {
+      try {
+        // Use existing main video task chain
+        const videoRendition = mediaEngine.getVideoRendition();
+        if (!videoRendition) {
+          throw new Error(
+            "Video rendition unavailable after checking videoRendition exists",
+          );
+        }
 
-    // Create main video input
-    const { BufferedSeekingInput } = await import("../BufferedSeekingInput.js");
-    const { EFMedia } = await import("../../EFMedia.js");
+        const segmentId = mediaEngine.computeSegmentId(
+          desiredSeekTimeMs,
+          videoRendition,
+        );
+        if (segmentId === undefined) {
+          span.setAttribute("result", "no-segment-id");
+          return undefined;
+        }
 
-    const startTimeOffsetMs = videoRendition?.startTimeOffsetMs;
+        span.setAttribute("segmentId", segmentId);
+
+        // Get cached main video input or create new one
+        const mainInput = await mainVideoInputCache.getOrCreateInput(
+          mediaEngine.src,
+          segmentId,
+          videoRendition.id,
+          async () => {
+            // Fetch main video segment (will be cached at mediaEngine level)
+            const [initSegment, mediaSegment] = await Promise.all([
+              mediaEngine.fetchInitSegment(videoRendition, signal),
+              mediaEngine.fetchMediaSegment(segmentId, videoRendition, signal),
+            ]);
+
+            if (!initSegment || !mediaSegment) {
+              return undefined;
+            }
+            signal.throwIfAborted();
+
+            const startTimeOffsetMs = videoRendition?.startTimeOffsetMs;
+
+            return new BufferedSeekingInput(
+              await new Blob([initSegment, mediaSegment]).arrayBuffer(),
+              {
+                videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
+                audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
+                startTimeOffsetMs,
+              },
+            );
+          },
+        );
 
-    const mainInput = new BufferedSeekingInput(
-      await new Blob([initSegment, mediaSegment]).arrayBuffer(),
-      {
-        videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
-        audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
-        startTimeOffsetMs,
-      },
-    );
+        if (!mainInput) {
+          span.setAttribute("result", "no-segments");
+          return undefined;
+        }
 
-    const videoTrack = await mainInput.getFirstVideoTrack();
-    if (!videoTrack) return undefined;
+        if (signal.aborted) {
+          span.setAttribute("result", "aborted-after-input");
+          return undefined;
+        }
 
-    signal.throwIfAborted();
+        const videoTrack = await mainInput.getFirstVideoTrack();
+        if (!videoTrack) {
+          span.setAttribute("result", "no-video-track");
+          return undefined;
+        }
 
-    const sample = (await mainInput.seek(
-      videoTrack.id,
-      desiredSeekTimeMs,
-    )) as unknown as VideoSample | undefined;
+        if (signal.aborted) {
+          span.setAttribute("result", "aborted-after-track");
+          return undefined;
+        }
 
-    return sample;
-  } catch (error) {
-    if (signal.aborted) return undefined;
-    throw error;
-  }
+        const sample = (await mainInput.seek(
+          videoTrack.id,
+          desiredSeekTimeMs,
+        )) as unknown as VideoSample | undefined;
+
+        span.setAttribute("result", sample ? "success" : "no-sample");
+        return sample;
+      } catch (error) {
+        if (signal.aborted) {
+          span.setAttribute("result", "aborted");
+          return undefined;
+        }
+        throw error;
+      }
+    },
+  );
 }
 
 /**

package/src/elements/EFMedia.browsertest.ts

@@ -1,6 +1,5 @@
 import { css } from "lit";
 import { customElement } from "lit/decorators.js";
-import type { VideoSample } from "mediabunny";
 import { afterEach, beforeEach, describe, vi } from "vitest";
 import { test as baseTest } from "../../test/useMSW.js";
 
@@ -106,9 +105,8 @@ describe("JIT Media Engine", () => {
       jitVideo,
       expect,
     }) => {
-      timegroup.currentTime = 2.2;
-      await timegroup.seekTask.taskComplete;
-      const sample = await jitVideo.unifiedVideoSeekTask.taskComplete;
+      await timegroup.seek(2200);
+      const sample = jitVideo.unifiedVideoSeekTask.value;
       expect(sample?.timestamp).toBeCloseTo(2.2, 1);
     });
   });
@@ -148,9 +146,8 @@ describe("JIT Media Engine", () => {
       expect,
     }) => {
       await timegroup.waitForMediaDurations();
-      timegroup.currentTimeMs = 3_000;
-      await timegroup.seekTask.taskComplete;
-      const frame = await jitVideo.unifiedVideoSeekTask.taskComplete;
+      await timegroup.seek(3000);
+      const frame = jitVideo.unifiedVideoSeekTask.value;
       expect(frame?.timestamp).toBeCloseTo(3, 1);
     });
 
@@ -160,9 +157,8 @@ describe("JIT Media Engine", () => {
       expect,
     }) => {
       await timegroup.waitForMediaDurations();
-      timegroup.currentTimeMs = 5_000;
-      await timegroup.seekTask.taskComplete;
-      const frame = await jitVideo.unifiedVideoSeekTask.taskComplete;
+      await timegroup.seek(5000);
+      const frame = jitVideo.unifiedVideoSeekTask.value;
       expect(frame?.timestamp).toBeCloseTo(5, 1);
     });
 
@@ -172,17 +168,14 @@ describe("JIT Media Engine", () => {
       expect,
     }) => {
       await timegroup.waitForMediaDurations();
-      timegroup.currentTimeMs = 0;
 
       // Test seeking in larger increments to avoid CI timeouts
       // while still validating incremental seeking works
       const testPoints = [0, 500, 1000, 1500, 2000, 2500, 3000];
-      let frame: VideoSample | undefined;
 
       for (const timeMs of testPoints) {
-        timegroup.currentTimeMs = timeMs;
-        await timegroup.seekTask.taskComplete;
-        frame = await jitVideo.unifiedVideoSeekTask.taskComplete;
+        await timegroup.seek(timeMs);
+        const frame = jitVideo.unifiedVideoSeekTask.value;
         expect(frame).toBeDefined();
         expect(frame?.timestamp).toBeCloseTo(timeMs / 1000, 1);
       }

package/src/elements/EFMedia.ts

@@ -1,6 +1,10 @@
+import { provide } from "@lit/context";
 import { css, LitElement, type PropertyValueMap } from "lit";
 import { property, state } from "lit/decorators.js";
 import { isContextMixin } from "../gui/ContextMixin.js";
+import type { ControllableInterface } from "../gui/Controllable.js";
+import { efContext } from "../gui/efContext.js";
+import { withSpan } from "../otel/tracingHelpers.js";
 import type { AudioSpan } from "../transcoding/types/index.ts";
 import { UrlGenerator } from "../transcoding/utils/UrlGenerator.ts";
 import { makeAudioBufferTask } from "./EFMedia/audioTasks/makeAudioBufferTask.ts";
@@ -16,6 +20,7 @@ import { makeMediaEngineTask } from "./EFMedia/tasks/makeMediaEngineTask.ts";
 import { EFSourceMixin } from "./EFSourceMixin.js";
 import { EFTemporal } from "./EFTemporal.js";
 import { FetchMixin } from "./FetchMixin.js";
+import { renderTemporalAudio } from "./renderTemporalAudio.js";
 import { EFTargetable } from "./TargetController.ts";
 
 // EF_FRAMEGEN is a global instance created in EF_FRAMEGEN.ts
@@ -46,6 +51,11 @@ export class EFMedia extends EFTargetable(
     assetType: "isobmff_files",
   }),
 ) {
+  @provide({ context: efContext })
+  get efContext(): ControllableInterface | null {
+    return this.rootTimegroup ?? this;
+  }
+
   // Sample buffer size configuration
   static readonly VIDEO_SAMPLE_BUFFER_SIZE = 30;
   static readonly AUDIO_SAMPLE_BUFFER_SIZE = 120;
@@ -248,13 +258,6 @@ export class EFMedia extends EFTargetable(
         }
       }
     }
-
-    // if (
-    //   changedProperties.has("currentTime") ||
-    //   changedProperties.has("ownCurrentTimeMs")
-    // ) {
-    //   updateAnimations(this);
-    // }
   }
 
   get hasOwnDuration() {
@@ -290,6 +293,49 @@ export class EFMedia extends EFTargetable(
     toMs: number,
     signal: AbortSignal = new AbortController().signal,
   ): Promise<AudioSpan | undefined> {
-    return fetchAudioSpanningTime(this, fromMs, toMs, signal);
+    return withSpan(
+      "media.fetchAudioSpanningTime",
+      {
+        elementId: this.id || "unknown",
+        tagName: this.tagName.toLowerCase(),
+        fromMs,
+        toMs,
+        durationMs: toMs - fromMs,
+        src: this.src || "none",
+      },
+      undefined,
+      async () => {
+        return fetchAudioSpanningTime(this, fromMs, toMs, signal);
+      },
+    );
+  }
+
+  /**
+   * Wait for media engine to load and determine duration
+   * Ensures media is ready for playback
+   */
+  async waitForMediaDurations(): Promise<void> {
+    if (this.mediaEngineTask.value) {
+      return;
+    }
+    await this.mediaEngineTask.run();
+  }
+
+  /**
+   * Returns media elements for playback audio rendering
+   * For standalone media, returns [this]; for timegroups, returns all descendants
+   * Used by PlaybackController for audio-driven playback
+   */
+  getMediaElements(): EFMedia[] {
+    return [this];
+  }
+
+  /**
+   * Render audio buffer for playback
+   * Called by PlaybackController during live playback
+   * Delegates to shared renderTemporalAudio utility for consistent behavior
+   */
+  async renderAudio(fromMs: number, toMs: number): Promise<AudioBuffer> {
+    return renderTemporalAudio(this, fromMs, toMs);
   }
 }

package/src/elements/EFSurface.browsertest.ts

@@ -26,7 +26,7 @@ const surfaceTest = baseTest.extend<{
     const container = document.createElement("div");
     render(
       html`
-        <ef-configuration api-host="http://localhost:63315">
+        <ef-configuration api-host="http://localhost:63315" signing-url="">
           <ef-preview>
             <ef-timegroup id="tg" mode="sequence" class="relative h-[360px] w-[640px] overflow-hidden bg-black">
               <ef-video id="vid" src="bars-n-tone.mp4" style="width: 100%; height: 100%;"></ef-video>
@@ -38,8 +38,6 @@ const surfaceTest = baseTest.extend<{
       container,
     );
     document.body.appendChild(container);
-    const configuration = container.querySelector("ef-configuration") as any;
-    configuration.signingURL = "";
     const tg = container.querySelector("#tg") as EFTimegroup;
     await tg.updateComplete;
     await use(tg);
@@ -103,7 +101,7 @@ describe("EFSurface", () => {
       const container = document.createElement("div");
       render(
         html`
-        <ef-configuration api-host="http://localhost:63315">
+        <ef-configuration api-host="http://localhost:63315" signing-url="">
           <ef-preview>
             <ef-timegroup mode="sequence" class="relative h-[360px] w-[640px] overflow-hidden bg-black">
               <ef-video id="v" src="bars-n-tone.mp4" style="width: 100%; height: 100%;"></ef-video>
@@ -116,8 +114,6 @@ describe("EFSurface", () => {
         container,
       );
       document.body.appendChild(container);
-      const configuration = container.querySelector("ef-configuration") as any;
-      configuration.signingURL = "";
       const timegroup = container.querySelector("ef-timegroup") as EFTimegroup;
       const video = container.querySelector("ef-video") as EFVideo;
       const s1 = container.querySelector("#s1") as unknown as EFSurface;

package/src/elements/EFSurface.ts

@@ -25,6 +25,7 @@ export class EFSurface extends LitElement {
   canvasRef = createRef<HTMLCanvasElement>();
 
   // @ts-expect-error controller is intentionally not referenced directly
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: Used for side effects
   #targetController: TargetController = new TargetController(this);
 
   @state()