@editframe/elements 0.20.4-beta.0 → 0.23.6-beta.0

package/src/elements/EFVideo.ts

@@ -1,11 +1,12 @@
 import { Task } from "@lit/task";
+import { context, trace } from "@opentelemetry/api";
 import debug from "debug";
 import { css, html, type PropertyValueMap } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
 import { createRef, ref } from "lit/directives/ref.js";
-
 import { DelayedLoadingState } from "../DelayedLoadingState.js";
 import { TWMixin } from "../gui/TWMixin.js";
+import { withSpan, withSpanSync } from "../otel/tracingHelpers.js";
 import { makeScrubVideoBufferTask } from "./EFMedia/videoTasks/makeScrubVideoBufferTask.ts";
 import { makeScrubVideoInitSegmentFetchTask } from "./EFMedia/videoTasks/makeScrubVideoInitSegmentFetchTask.ts";
 import { makeScrubVideoInputTask } from "./EFMedia/videoTasks/makeScrubVideoInputTask.ts";
@@ -15,6 +16,7 @@ import { makeScrubVideoSegmentIdTask } from "./EFMedia/videoTasks/makeScrubVideo
 import { makeUnifiedVideoSeekTask } from "./EFMedia/videoTasks/makeUnifiedVideoSeekTask.ts";
 import { makeVideoBufferTask } from "./EFMedia/videoTasks/makeVideoBufferTask.ts";
 import { EFMedia } from "./EFMedia.js";
+import { updateAnimations } from "./updateAnimations.js";
 
 // EF_FRAMEGEN is a global instance created in EF_FRAMEGEN.ts
 declare global {
@@ -203,11 +205,46 @@ export class EFVideo extends TWMixin(EFMedia) {
       console.error("frameTask error", error);
     },
     onComplete: () => {},
-    task: async ([_desiredSeekTimeMs]) => {
-      this.unifiedVideoSeekTask.run();
-      await this.unifiedVideoSeekTask.taskComplete;
-      this.paintTask.run();
-      await this.paintTask.taskComplete;
+    task: async ([_desiredSeekTimeMs], { signal }) => {
+      const t0 = performance.now();
+
+      await withSpan(
+        "video.frameTask",
+        {
+          elementId: this.id || "unknown",
+          desiredSeekTimeMs: _desiredSeekTimeMs,
+          src: this.src || "none",
+        },
+        undefined,
+        async (span) => {
+          const t1 = performance.now();
+          span.setAttribute("preworkMs", t1 - t0);
+
+          this.unifiedVideoSeekTask.run();
+          const t2 = performance.now();
+          span.setAttribute("seekRunMs", t2 - t1);
+
+          await this.unifiedVideoSeekTask.taskComplete;
+          const t3 = performance.now();
+          span.setAttribute("seekAwaitMs", t3 - t2);
+          if (signal.aborted) {
+            span.setAttribute("aborted", true);
+            return;
+          }
+
+          this.paint(this.desiredSeekTimeMs, span);
+
+          if (!this.parentTimegroup) {
+            updateAnimations(this);
+          }
+
+          const t4 = performance.now();
+          this.paint(_desiredSeekTimeMs, span);
+          const t5 = performance.now();
+          span.setAttribute("paintMs", t5 - t4);
+          span.setAttribute("totalFrameMs", t5 - t0);
+        },
+      );
     },
   });
 
@@ -244,64 +281,92 @@ export class EFVideo extends TWMixin(EFMedia) {
     };
   }
 
-  paintTask = new Task(this, {
-    autoRun: false,
-    args: () => [this.desiredSeekTimeMs] as const,
-    onError: (error) => {
-      console.error("paintTask error", error);
-    },
-    onComplete: () => {},
-    task: async ([_seekToMs], { signal }) => {
-      // Check if we're in production rendering mode vs preview mode
-      const isProductionRendering = this.isInProductionRenderingMode();
-
-      // Unified video system: smart routing to scrub or main, with background upgrades
-      try {
-        await this.unifiedVideoSeekTask.taskComplete;
-        const videoSample = this.unifiedVideoSeekTask.value;
-
-        if (videoSample) {
-          const videoFrame = videoSample.toVideoFrame();
-          try {
-            this.displayFrame(videoFrame, _seekToMs);
-          } finally {
-            videoFrame.close();
+  /**
+   * Paint the current video frame to canvas
+   * Called by frameTask after seek is complete
+   */
+  paint(seekToMs: number, parentSpan?: any): void {
+    const parentContext = parentSpan
+      ? trace.setSpan(context.active(), parentSpan)
+      : undefined;
+
+    withSpanSync(
+      "video.paint",
+      {
+        elementId: this.id || "unknown",
+        seekToMs,
+        src: this.src || "none",
+      },
+      parentContext,
+      (span) => {
+        const t0 = performance.now();
+
+        // Check if we're in production rendering mode vs preview mode
+        const isProductionRendering = this.isInProductionRenderingMode();
+        const t1 = performance.now();
+        span.setAttribute("isProductionRendering", isProductionRendering);
+        span.setAttribute("modeCheckMs", t1 - t0);
+
+        // Unified video system: smart routing to scrub or main, with background upgrades
+        // Note: frameTask guarantees unifiedVideoSeekTask is complete before calling paint
+        try {
+          const t2 = performance.now();
+          const videoSample = this.unifiedVideoSeekTask.value;
+          span.setAttribute("hasVideoSample", !!videoSample);
+          span.setAttribute("valueAccessMs", t2 - t1);
+
+          if (videoSample) {
+            const t3 = performance.now();
+            const videoFrame = videoSample.toVideoFrame();
+            const t4 = performance.now();
+            span.setAttribute("toVideoFrameMs", t4 - t3);
+
+            try {
+              const t5 = performance.now();
+              this.displayFrame(videoFrame, seekToMs, span);
+              const t6 = performance.now();
+              span.setAttribute("displayFrameMs", t6 - t5);
+            } finally {
+              videoFrame.close();
+            }
           }
+        } catch (error) {
+          console.warn("Unified video pipeline error:", error);
         }
-      } catch (error) {
-        console.warn("Unified video pipeline error:", error);
-      }
 
-      // EF_FRAMEGEN-aware rendering mode detection
-      if (!isProductionRendering) {
-        // Preview mode: skip rendering during initialization to prevent artifacts
-        if (
-          !this.rootTimegroup ||
-          (this.rootTimegroup.currentTimeMs === 0 &&
-            this.desiredSeekTimeMs === 0)
-        ) {
-          return; // Skip initialization frame in preview mode
-        }
-        // Preview mode: proceed with rendering
-      } else {
-        // Production rendering mode: only render when EF_FRAMEGEN has explicitly started frame rendering
-        // This prevents initialization frames before the actual render sequence begins
-        if (!this.rootTimegroup) {
-          return;
-        }
+        // EF_FRAMEGEN-aware rendering mode detection
+        if (!isProductionRendering) {
+          // Preview mode: skip rendering during initialization to prevent artifacts
+          if (
+            !this.rootTimegroup ||
+            (this.rootTimegroup.currentTimeMs === 0 &&
+              this.desiredSeekTimeMs === 0)
+          ) {
+            span.setAttribute("skipped", "preview-initialization");
+            return; // Skip initialization frame in preview mode
+          }
+          // Preview mode: proceed with rendering
+        } else {
+          // Production rendering mode: only render when EF_FRAMEGEN has explicitly started frame rendering
+          // This prevents initialization frames before the actual render sequence begins
+          if (!this.rootTimegroup) {
+            span.setAttribute("skipped", "no-root-timegroup");
+            return;
+          }
 
-        if (!this.isFrameRenderingActive()) {
-          return; // Wait for EF_FRAMEGEN to start frame sequence
-        }
+          if (!this.isFrameRenderingActive()) {
+            span.setAttribute("skipped", "frame-rendering-not-active");
+            return; // Wait for EF_FRAMEGEN to start frame sequence
+          }
 
-        // Production mode: EF_FRAMEGEN has started frame sequence, proceed with rendering
-      }
+          // Production mode: EF_FRAMEGEN has started frame sequence, proceed with rendering
+        }
 
-      if (signal.aborted) {
-        return;
-      }
-    },
-  });
+        const tEnd = performance.now();
+        span.setAttribute("totalPaintMs", tEnd - t0);
+      },
+    );
+  }
 
   /**
    * Clear the canvas when element becomes inactive
@@ -318,54 +383,98 @@ export class EFVideo extends TWMixin(EFMedia) {
   /**
    * Display a video frame on the canvas
    */
-  displayFrame(frame: VideoFrame, seekToMs: number): number {
-    log("trace: displayFrame start", { seekToMs, frameFormat: frame.format });
-    if (!this.canvasElement) {
-      log("trace: displayFrame aborted - no canvas element");
-      throw new Error(
-        `Frame display failed: Canvas element is not available at time ${seekToMs}ms. The video component may not be properly initialized.`,
-      );
-    }
-
-    const ctx = this.canvasElement.getContext("2d");
-    if (!ctx) {
-      log("trace: displayFrame aborted - no canvas context");
-      throw new Error(
-        `Frame display failed: Unable to get 2D canvas context at time ${seekToMs}ms. This may indicate a browser compatibility issue or canvas corruption.`,
-      );
-    }
+  displayFrame(frame: VideoFrame, seekToMs: number, parentSpan?: any): void {
+    const parentContext = parentSpan
+      ? trace.setSpan(context.active(), parentSpan)
+      : undefined;
+
+    withSpanSync(
+      "video.displayFrame",
+      {
+        elementId: this.id || "unknown",
+        seekToMs,
+        format: frame.format || "unknown",
+        width: frame.codedWidth,
+        height: frame.codedHeight,
+      },
+      parentContext,
+      (span) => {
+        const t0 = performance.now();
 
-    if (frame?.codedWidth && frame?.codedHeight) {
-      if (
-        this.canvasElement.width !== frame.codedWidth ||
-        this.canvasElement.height !== frame.codedHeight
-      ) {
-        log("trace: updating canvas dimensions", {
-          width: frame.codedWidth,
-          height: frame.codedHeight,
+        log("trace: displayFrame start", {
+          seekToMs,
+          frameFormat: frame.format,
         });
-        this.canvasElement.width = frame.codedWidth;
-        this.canvasElement.height = frame.codedHeight;
-      }
-    }
 
-    if (frame.format === null) {
-      log("trace: displayFrame aborted - null frame format");
-      throw new Error(
-        `Frame display failed: Video frame has null format at time ${seekToMs}ms. This indicates corrupted or incompatible video data.`,
-      );
-    }
+        if (!this.canvasElement) {
+          log("trace: displayFrame aborted - no canvas element");
+          throw new Error(
+            `Frame display failed: Canvas element is not available at time ${seekToMs}ms. The video component may not be properly initialized.`,
+          );
+        }
+        const t1 = performance.now();
+        span.setAttribute("getCanvasMs", Math.round((t1 - t0) * 100) / 100);
+
+        const ctx = this.canvasElement.getContext("2d");
+        const t2 = performance.now();
+        span.setAttribute("getCtxMs", Math.round((t2 - t1) * 100) / 100);
+
+        if (!ctx) {
+          log("trace: displayFrame aborted - no canvas context");
+          throw new Error(
+            `Frame display failed: Unable to get 2D canvas context at time ${seekToMs}ms. This may indicate a browser compatibility issue or canvas corruption.`,
+          );
+        }
 
-    ctx.drawImage(
-      frame,
-      0,
-      0,
-      this.canvasElement.width,
-      this.canvasElement.height,
-    );
-    log("trace: frame drawn to canvas", { seekToMs });
+        let resized = false;
+        if (frame?.codedWidth && frame?.codedHeight) {
+          if (
+            this.canvasElement.width !== frame.codedWidth ||
+            this.canvasElement.height !== frame.codedHeight
+          ) {
+            log("trace: updating canvas dimensions", {
+              width: frame.codedWidth,
+              height: frame.codedHeight,
+            });
+            this.canvasElement.width = frame.codedWidth;
+            this.canvasElement.height = frame.codedHeight;
+            resized = true;
+            const t3 = performance.now();
+            span.setAttribute("resizeMs", Math.round((t3 - t2) * 100) / 100);
+          }
+        }
+        span.setAttribute("canvasResized", resized);
 
-    return seekToMs;
+        if (frame.format === null) {
+          log("trace: displayFrame aborted - null frame format");
+          throw new Error(
+            `Frame display failed: Video frame has null format at time ${seekToMs}ms. This indicates corrupted or incompatible video data.`,
+          );
+        }
+
+        const tDrawStart = performance.now();
+        ctx.drawImage(
+          frame,
+          0,
+          0,
+          this.canvasElement.width,
+          this.canvasElement.height,
+        );
+        const tDrawEnd = performance.now();
+        span.setAttribute(
+          "drawImageMs",
+          Math.round((tDrawEnd - tDrawStart) * 100) / 100,
+        );
+        span.setAttribute(
+          "totalDisplayMs",
+          Math.round((tDrawEnd - t0) * 100) / 100,
+        );
+        span.setAttribute("canvasWidth", this.canvasElement.width);
+        span.setAttribute("canvasHeight", this.canvasElement.height);
+
+        log("trace: frame drawn to canvas", { seekToMs });
+      },
+    );
   }
 
   /**
@@ -419,6 +528,18 @@ export class EFVideo extends TWMixin(EFMedia) {
     return this.unifiedVideoSeekTask;
   }
 
+  /**
+   * Helper method for tests: wait for the current frame to be ready
+   * This encapsulates the complexity of ensuring the video has updated
+   * and its frameTask has completed.
+   *
+   * @returns Promise that resolves when the frame is ready
+   */
+  async waitForFrameReady(): Promise<void> {
+    await this.updateComplete;
+    await this.frameTask.run();
+  }
+
   /**
    * Clean up resources when component is disconnected
    */
@@ -428,6 +549,13 @@ export class EFVideo extends TWMixin(EFMedia) {
     // Clean up delayed loading state
     this.delayedLoadingState.clearAllLoading();
   }
+
+  didBecomeRoot() {
+    super.didBecomeRoot();
+  }
+  didBecomeChild() {
+    super.didBecomeChild();
+  }
 }
 
 declare global {

package/src/elements/FetchContext.browsertest.ts

@@ -10,9 +10,12 @@ import "../gui/EFConfiguration.js";
 const test = baseTest.extend({});
 
 describe("URL Token Deduplication", () => {
-  test("multiple EFMedia elements with same src should share URL tokens", async ({
+  test.skip("multiple EFMedia elements with same src should share URL tokens", async ({
     expect,
   }) => {
+    // TODO: This test is intentionally skipped because it documents a known issue where
+    // URL token requests are not properly deduplicated across multiple EFMedia elements
+    // with the same src. Currently makes 2 token requests instead of 1.
     // Mock fetch to track token requests
     const originalFetch = window.fetch;
     const tokenRequests: string[] = [];
@@ -299,7 +302,9 @@ describe("URL Token Deduplication", () => {
       expect(tokenRequests.length).toBe(1);
 
       // Cleanup
-      containers.forEach((container) => container.remove());
+      containers.forEach((container) => {
+        container.remove();
+      });
     } finally {
       window.fetch = originalFetch;
     }

package/src/elements/TargetController.browsertest.ts

@@ -22,6 +22,7 @@ class TargetableTest extends EFTargetable(LitElement) {
 @customElement("targeter-test")
 class TargeterTest extends LitElement {
   // @ts-expect-error this controller is needed, but never referenced
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: Used for side effects
   private targetController: TargetController = new TargetController(this);
 
   @state()

package/src/elements/TargetController.ts

@@ -176,6 +176,7 @@ export class TargetController implements ReactiveController {
       this.disconnectFromTarget();
       this.host.targetElement = newTarget ?? (null as Element | null);
       this.connectToTarget();
+      this.host.requestUpdate("targetElement");
     }
   }
 

package/src/elements/renderTemporalAudio.ts

@@ -0,0 +1,108 @@
+import type { EFMedia } from "./EFMedia.js";
+
+interface TemporalAudioHost {
+  startTimeMs: number;
+  endTimeMs: number;
+  durationMs: number;
+  getMediaElements(): EFMedia[];
+  waitForMediaDurations?(): Promise<void>;
+}
+
+export async function renderTemporalAudio(
+  host: TemporalAudioHost,
+  fromMs: number,
+  toMs: number,
+): Promise<AudioBuffer> {
+  const durationMs = toMs - fromMs;
+  const duration = durationMs / 1000;
+  const exactSamples = 48000 * duration;
+  const aacFrames = exactSamples / 1024;
+  const alignedFrames = Math.round(aacFrames);
+  const contextSize = alignedFrames * 1024;
+
+  if (contextSize <= 0) {
+    throw new Error(
+      `Duration must be greater than 0 when rendering audio. ${contextSize}ms`,
+    );
+  }
+
+  const audioContext = new OfflineAudioContext(2, contextSize, 48000);
+
+  if (host.waitForMediaDurations) {
+    await host.waitForMediaDurations();
+  }
+
+  const abortController = new AbortController();
+
+  await Promise.all(
+    host.getMediaElements().map(async (mediaElement) => {
+      if (mediaElement.mute) {
+        return;
+      }
+
+      const mediaStartsBeforeEnd = mediaElement.startTimeMs <= toMs;
+      const mediaEndsAfterStart = mediaElement.endTimeMs >= fromMs;
+      const mediaOverlaps = mediaStartsBeforeEnd && mediaEndsAfterStart;
+      if (!mediaOverlaps) {
+        return;
+      }
+
+      const mediaLocalFromMs = Math.max(0, fromMs - mediaElement.startTimeMs);
+      const mediaLocalToMs = Math.min(
+        mediaElement.endTimeMs - mediaElement.startTimeMs,
+        toMs - mediaElement.startTimeMs,
+      );
+
+      if (mediaLocalFromMs >= mediaLocalToMs) {
+        return;
+      }
+
+      const sourceInMs =
+        mediaElement.sourceInMs || mediaElement.trimStartMs || 0;
+      const mediaSourceFromMs = mediaLocalFromMs + sourceInMs;
+      const mediaSourceToMs = mediaLocalToMs + sourceInMs;
+
+      const audio = await mediaElement.fetchAudioSpanningTime(
+        mediaSourceFromMs,
+        mediaSourceToMs,
+        abortController.signal,
+      );
+      if (!audio) {
+        return;
+      }
+
+      const bufferSource = audioContext.createBufferSource();
+      bufferSource.buffer = await audioContext.decodeAudioData(
+        await audio.blob.arrayBuffer(),
+      );
+      bufferSource.connect(audioContext.destination);
+
+      const ctxStartMs = Math.max(0, mediaElement.startTimeMs - fromMs);
+
+      const requestedSourceFromMs = mediaSourceFromMs;
+      const actualSourceStartMs = audio.startMs;
+      const offsetInBufferMs = requestedSourceFromMs - actualSourceStartMs;
+
+      const safeOffsetMs = Math.max(0, offsetInBufferMs);
+
+      const requestedDurationMs = mediaSourceToMs - mediaSourceFromMs;
+      const availableAudioMs = audio.endMs - audio.startMs;
+      const actualDurationMs = Math.min(
+        requestedDurationMs,
+        availableAudioMs - safeOffsetMs,
+      );
+
+      if (actualDurationMs <= 0) {
+        return;
+      }
+
+      bufferSource.start(
+        ctxStartMs / 1000,
+        safeOffsetMs / 1000,
+        actualDurationMs / 1000,
+      );
+    }),
+  );
+
+  return audioContext.startRendering();
+}