@editframe/elements 0.20.3-beta.0 → 0.21.0-beta.0

package/src/elements/EFMedia/BufferedSeekingInput.ts

@@ -8,6 +8,7 @@ import {
   MP4,
   VideoSampleSink,
 } from "mediabunny";
+import { withSpan } from "../../otel/tracingHelpers.js";
 import { type MediaSample, SampleBuffer } from "../SampleBuffer";
 import { roundToMilliseconds } from "./shared/PrecisionUtils";
 
@@ -177,26 +178,39 @@ export class BufferedSeekingInput {
   }
 
   async seek(trackId: number, timeMs: number) {
-    // Apply timeline offset to map user timeline to media timeline
-    const mediaTimeMs = timeMs + this.startTimeOffsetMs;
-
-    // Round using consistent precision handling
-    const roundedMediaTimeMs = roundToMilliseconds(mediaTimeMs);
-
-    // Serialize seek operations per track (but don't block iterator creation)
-    const existingSeek = this.trackSeekPromises.get(trackId);
-    if (existingSeek) {
-      await existingSeek;
-    }
+    return withSpan(
+      "bufferedInput.seek",
+      {
+        trackId,
+        timeMs,
+        startTimeOffsetMs: this.startTimeOffsetMs,
+      },
+      undefined,
+      async (span) => {
+        // Apply timeline offset to map user timeline to media timeline
+        const mediaTimeMs = timeMs + this.startTimeOffsetMs;
+
+        // Round using consistent precision handling
+        const roundedMediaTimeMs = roundToMilliseconds(mediaTimeMs);
+        span.setAttribute("roundedMediaTimeMs", roundedMediaTimeMs);
+
+        // Serialize seek operations per track (but don't block iterator creation)
+        const existingSeek = this.trackSeekPromises.get(trackId);
+        if (existingSeek) {
+          span.setAttribute("waitedForExistingSeek", true);
+          await existingSeek;
+        }
 
-    const seekPromise = this.seekSafe(trackId, roundedMediaTimeMs);
-    this.trackSeekPromises.set(trackId, seekPromise);
+        const seekPromise = this.seekSafe(trackId, roundedMediaTimeMs);
+        this.trackSeekPromises.set(trackId, seekPromise);
 
-    try {
-      return await seekPromise;
-    } finally {
-      this.trackSeekPromises.delete(trackId);
-    }
+        try {
+          return await seekPromise;
+        } finally {
+          this.trackSeekPromises.delete(trackId);
+        }
+      },
+    );
   }
 
   private async resetIterator(track: InputTrack) {
@@ -224,90 +238,193 @@ export class BufferedSeekingInput {
   #seekLock?: PromiseWithResolvers<void>;
 
   private async seekSafe(trackId: number, timeMs: number) {
-    if (this.#seekLock) {
-      await this.#seekLock.promise;
-    }
-    const seekLock = Promise.withResolvers<void>();
-    this.#seekLock = seekLock;
-
-    try {
-      const track = await this.getTrack(trackId);
-      const trackBuffer = this.getTrackBuffer(track);
-
-      const roundedTimeMs = roundToMilliseconds(timeMs);
-      const firstTimestampMs = roundToMilliseconds(
-        (await track.getFirstTimestamp()) * 1000,
-      );
-
-      if (roundedTimeMs < firstTimestampMs) {
-        console.error("Seeking outside bounds of input", {
-          roundedTimeMs,
-          firstTimestampMs,
-        });
-        throw new NoSample(
-          `Seeking outside bounds of input ${roundedTimeMs} < ${firstTimestampMs}`,
-        );
-      }
-
-      // Check if we need to reset iterator for seeks outside current buffer range
-      const bufferContents = trackBuffer.getContents();
-      if (bufferContents.length > 0) {
-        const bufferStartMs = roundToMilliseconds(
-          trackBuffer.firstTimestamp * 1000,
-        );
-
-        if (roundedTimeMs < bufferStartMs) {
-          await this.resetIterator(track);
-        }
-      }
-
-      const alreadyInBuffer = trackBuffer.find(timeMs);
-      if (alreadyInBuffer) return alreadyInBuffer;
-
-      const iterator = this.getTrackIterator(track);
-      while (true) {
-        const { done, value: decodedSample } = await iterator.next();
-
-        if (decodedSample) {
-          trackBuffer.push(decodedSample);
-        }
-        const foundSample = trackBuffer.find(roundedTimeMs);
-        if (foundSample) {
-          return foundSample;
-        }
-        if (done) {
-          break;
+    return withSpan(
+      "bufferedInput.seekSafe",
+      {
+        trackId,
+        timeMs,
+      },
+      undefined,
+      async (span) => {
+        if (this.#seekLock) {
+          span.setAttribute("waitedForSeekLock", true);
+          await this.#seekLock.promise;
         }
-      }
-
-      // Check if we're seeking to the exact end of the track (legitimate use case)
-      const finalBufferContents = trackBuffer.getContents();
-      if (finalBufferContents.length > 0) {
-        const lastSample = finalBufferContents[finalBufferContents.length - 1];
-        const lastSampleEndMs = roundToMilliseconds(
-          ((lastSample?.timestamp || 0) + (lastSample?.duration || 0)) * 1000,
-        );
-
-        // Only return last sample if seeking to exactly the track duration
-        // (end of video) AND we have the final segment loaded
-        const trackDurationMs = (await track.computeDuration()) * 1000;
-        const isSeekingToTrackEnd =
-          roundToMilliseconds(timeMs) === roundToMilliseconds(trackDurationMs);
-        const isAtEndOfTrack = roundToMilliseconds(timeMs) >= lastSampleEndMs;
-
-        if (isSeekingToTrackEnd && isAtEndOfTrack) {
-          return lastSample;
+        const seekLock = Promise.withResolvers<void>();
+        this.#seekLock = seekLock;
+
+        try {
+          const track = await this.getTrack(trackId);
+          span.setAttribute("trackType", track.type);
+
+          const trackBuffer = this.getTrackBuffer(track);
+
+          const roundedTimeMs = roundToMilliseconds(timeMs);
+          const firstTimestampMs = roundToMilliseconds(
+            (await track.getFirstTimestamp()) * 1000,
+          );
+          span.setAttribute("firstTimestampMs", firstTimestampMs);
+
+          if (roundedTimeMs < firstTimestampMs) {
+            console.error("Seeking outside bounds of input", {
+              roundedTimeMs,
+              firstTimestampMs,
+            });
+            throw new NoSample(
+              `Seeking outside bounds of input ${roundedTimeMs} < ${firstTimestampMs}`,
+            );
+          }
+
+          // Check if we need to reset iterator for seeks outside current buffer range
+          const bufferContents = trackBuffer.getContents();
+          span.setAttribute("bufferContentsLength", bufferContents.length);
+
+          if (bufferContents.length > 0) {
+            const bufferStartMs = roundToMilliseconds(
+              trackBuffer.firstTimestamp * 1000,
+            );
+            span.setAttribute("bufferStartMs", bufferStartMs);
+
+            if (roundedTimeMs < bufferStartMs) {
+              span.setAttribute("resetIterator", true);
+              await this.resetIterator(track);
+            }
+          }
+
+          const alreadyInBuffer = trackBuffer.find(timeMs);
+          if (alreadyInBuffer) {
+            span.setAttribute("foundInBuffer", true);
+            span.setAttribute("bufferSize", trackBuffer.length);
+            const contents = trackBuffer.getContents();
+            if (contents.length > 0) {
+              span.setAttribute(
+                "bufferTimestamps",
+                contents
+                  .map((s) => Math.round((s.timestamp || 0) * 1000))
+                  .slice(0, 10)
+                  .join(","),
+              );
+            }
+            return alreadyInBuffer;
+          }
+
+          // Buffer miss - record buffer state
+          span.setAttribute("foundInBuffer", false);
+          span.setAttribute("bufferSize", trackBuffer.length);
+          span.setAttribute("requestedTimeMs", Math.round(timeMs));
+
+          const contents = trackBuffer.getContents();
+          if (contents.length > 0) {
+            const firstSample = contents[0];
+            const lastSample = contents[contents.length - 1];
+            if (firstSample && lastSample) {
+              const bufferStartMs = Math.round(
+                (firstSample.timestamp || 0) * 1000,
+              );
+              const bufferEndMs = Math.round(
+                ((lastSample.timestamp || 0) + (lastSample.duration || 0)) *
+                  1000,
+              );
+              span.setAttribute("bufferStartMs", bufferStartMs);
+              span.setAttribute("bufferEndMs", bufferEndMs);
+              span.setAttribute(
+                "bufferRangeMs",
+                `${bufferStartMs}-${bufferEndMs}`,
+              );
+            }
+          }
+
+          const iterator = this.getTrackIterator(track);
+          let iterationCount = 0;
+          const decodeStart = performance.now();
+
+          while (true) {
+            iterationCount++;
+            const iterStart = performance.now();
+            const { done, value: decodedSample } = await iterator.next();
+            const iterEnd = performance.now();
+
+            // Record individual iteration timing for first 5 iterations
+            if (iterationCount <= 5) {
+              span.setAttribute(
+                `iter${iterationCount}Ms`,
+                Math.round((iterEnd - iterStart) * 100) / 100,
+              );
+            }
+
+            if (decodedSample) {
+              trackBuffer.push(decodedSample);
+              if (iterationCount <= 5) {
+                span.setAttribute(
+                  `iter${iterationCount}Timestamp`,
+                  Math.round((decodedSample.timestamp || 0) * 1000),
+                );
+              }
+            }
+
+            const foundSample = trackBuffer.find(roundedTimeMs);
+            if (foundSample) {
+              const decodeEnd = performance.now();
+              span.setAttribute("iterationCount", iterationCount);
+              span.setAttribute(
+                "decodeMs",
+                Math.round((decodeEnd - decodeStart) * 100) / 100,
+              );
+              span.setAttribute(
+                "avgIterMs",
+                Math.round(((decodeEnd - decodeStart) / iterationCount) * 100) /
+                  100,
+              );
+              span.setAttribute("foundSample", true);
+              span.setAttribute(
+                "foundTimestamp",
+                Math.round((foundSample.timestamp || 0) * 1000),
+              );
+              return foundSample;
+            }
+            if (done) {
+              break;
+            }
+          }
+
+          span.setAttribute("iterationCount", iterationCount);
+          span.setAttribute("reachedEnd", true);
+
+          // Check if we're seeking to the exact end of the track (legitimate use case)
+          const finalBufferContents = trackBuffer.getContents();
+          if (finalBufferContents.length > 0) {
+            const lastSample =
+              finalBufferContents[finalBufferContents.length - 1];
+            const lastSampleEndMs = roundToMilliseconds(
+              ((lastSample?.timestamp || 0) + (lastSample?.duration || 0)) *
+                1000,
+            );
+
+            // Only return last sample if seeking to exactly the track duration
+            // (end of video) AND we have the final segment loaded
+            const trackDurationMs = (await track.computeDuration()) * 1000;
+            const isSeekingToTrackEnd =
+              roundToMilliseconds(timeMs) ===
+              roundToMilliseconds(trackDurationMs);
+            const isAtEndOfTrack =
+              roundToMilliseconds(timeMs) >= lastSampleEndMs;
+
+            if (isSeekingToTrackEnd && isAtEndOfTrack) {
+              span.setAttribute("returnedLastSample", true);
+              return lastSample;
+            }
+          }
+
+          // For all other cases (seeking within track but outside buffer range), throw error
+          // The caller should ensure the correct segment is loaded before seeking
+          throw new NoSample(
+            `Sample not found for time ${timeMs} in ${track.type} track ${trackId}`,
+          );
+        } finally {
+          this.#seekLock = undefined;
+          seekLock.resolve();
         }
-      }
-
-      // For all other cases (seeking within track but outside buffer range), throw error
-      // The caller should ensure the correct segment is loaded before seeking
-      throw new NoSample(
-        `Sample not found for time ${timeMs} in ${track.type} track ${trackId}`,
-      );
-    } finally {
-      this.#seekLock = undefined;
-      seekLock.resolve();
-    }
+      },
+    );
   }
 }

package/src/elements/EFMedia/JitMediaEngine.ts

@@ -215,15 +215,29 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
   async extractThumbnails(
     timestamps: number[],
   ): Promise<(ThumbnailResult | null)[]> {
-    const mainRendition = this.videoRendition;
-    const scrubRendition = this.getScrubVideoRendition();
-
-    const rendition = mainRendition || scrubRendition;
-
-    if (!rendition) {
+    // Use same rendition priority as video: try main rendition first for frame alignment
+    let rendition: VideoRendition;
+    try {
+      const mainRendition = this.getVideoRendition();
+      if (mainRendition) {
+        rendition = mainRendition;
+      } else {
+        const scrubRendition = this.getScrubVideoRendition();
+        if (scrubRendition) {
+          rendition = scrubRendition;
+        } else {
+          throw new Error("No video rendition available");
+        }
+      }
+    } catch (error) {
+      console.warn(
+        "JitMediaEngine: No video rendition available for thumbnails",
+        error,
+      );
       return timestamps.map(() => null);
     }
 
+    // Use shared thumbnail extraction logic
     return this.thumbnailExtractor.extractThumbnails(
       timestamps,
       rendition,

package/src/elements/EFMedia/audioTasks/makeAudioBufferTask.ts

@@ -42,15 +42,18 @@ export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
     task: async ([seekTimeMs], { signal }) => {
       // Skip buffering entirely in rendering mode
       if (EF_RENDERING()) {
-        return currentState;
+        return currentState; // Return existing state without any buffering activity
       }
 
+      // Get media engine to potentially override buffer configuration
       const mediaEngine = await getLatestMediaEngine(host, signal);
 
+      // Return existing state if no audio rendition available
       if (!mediaEngine.audioRendition) {
         return currentState;
       }
 
+      // Use media engine's buffer config, falling back to host properties
       const engineConfig = mediaEngine.getBufferConfig();
       const bufferDurationMs = engineConfig.audioBufferDurationMs;
       const maxParallelFetches = engineConfig.maxAudioBufferFetches;
@@ -90,7 +93,7 @@ export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
             const mediaEngine = await getLatestMediaEngine(host, signal);
             const audioRendition = mediaEngine.audioRendition;
             if (!audioRendition) {
-              throw new Error("No audio track available in source");
+              throw new Error("Audio rendition not available");
             }
             return audioRendition;
           },

package/src/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.ts

@@ -95,11 +95,6 @@ export function makeAudioFrequencyAnalysisTask(element: EFMedia) {
     task: async (_, { signal }) => {
       if (element.currentSourceTimeMs < 0) return null;
 
-      const mediaEngine = element.mediaEngineTask.value;
-      if (!mediaEngine?.audioRendition) {
-        return null;
-      }
-
       const currentTimeMs = element.currentSourceTimeMs;
 
       // Calculate exact audio window needed based on fftDecay and frame timing

package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.ts

@@ -14,8 +14,9 @@ export const makeAudioInitSegmentFetchTask = (
     onComplete: (_value) => {},
     task: async ([_mediaEngine], { signal }) => {
       const mediaEngine = await getLatestMediaEngine(host, signal);
+      const audioRendition = mediaEngine.getAudioRendition();
 
-      const audioRendition = mediaEngine.audioRendition;
+      // Return undefined if no audio rendition available (video-only asset)
       if (!audioRendition) {
         return undefined;
       }

package/src/elements/EFMedia/audioTasks/makeAudioInputTask.ts

@@ -6,7 +6,7 @@ import type { InputTask } from "../shared/MediaTaskUtils";
 export const makeAudioInputTask = (host: EFMedia): InputTask => {
   return new Task<
     readonly [ArrayBuffer | undefined, ArrayBuffer | undefined],
-    BufferedSeekingInput
+    BufferedSeekingInput | undefined
   >(host, {
     args: () =>
       [
@@ -18,21 +18,31 @@ export const makeAudioInputTask = (host: EFMedia): InputTask => {
     },
     onComplete: (_value) => {},
     task: async (_, { signal }) => {
+      const mediaEngine = await host.mediaEngineTask.taskComplete;
+      if (signal.aborted) return undefined;
+
+      const audioRendition = mediaEngine?.audioRendition;
+
+      // Return undefined if no audio rendition available (video-only asset)
+      if (!audioRendition) {
+        return undefined;
+      }
+
       const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
-      signal.throwIfAborted();
+      if (signal.aborted) return undefined;
+
       const segment = await host.audioSegmentFetchTask.taskComplete;
-      signal.throwIfAborted();
+      if (signal.aborted) return undefined;
 
       if (!initSegment || !segment) {
-        throw new Error("No audio track available in source");
+        return undefined;
       }
 
-      const mediaEngine = await host.mediaEngineTask.taskComplete;
-      const audioRendition = mediaEngine?.audioRendition;
-      const startTimeOffsetMs = audioRendition?.startTimeOffsetMs;
+      const startTimeOffsetMs = audioRendition.startTimeOffsetMs;
 
       const arrayBuffer = await new Blob([initSegment, segment]).arrayBuffer();
-      signal.throwIfAborted();
+      if (signal.aborted) return undefined;
+
       return new BufferedSeekingInput(arrayBuffer, {
         videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
         audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,

package/src/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.ts

@@ -18,26 +18,14 @@ export const makeAudioSegmentFetchTask = (
     onComplete: (_value) => {},
     task: async (_, { signal }) => {
       const mediaEngine = await getLatestMediaEngine(host, signal);
+      const segmentId = await host.audioSegmentIdTask.taskComplete;
+      const audioRendition = mediaEngine.getAudioRendition();
 
-      const audioRendition = mediaEngine.audioRendition;
-      if (!audioRendition) {
+      // Return undefined if no audio rendition or segment ID available (video-only asset)
+      if (!audioRendition || segmentId === undefined) {
         return undefined;
       }
 
-      const segmentId = await host.audioSegmentIdTask.taskComplete;
-      if (segmentId === undefined) {
-        const debugInfo = {
-          hasRendition: true,
-          segmentDurationMs: audioRendition.segmentDurationMs,
-          segmentDurationsMs: audioRendition.segmentDurationsMs?.length || 0,
-          desiredSeekTimeMs: host.desiredSeekTimeMs,
-          intrinsicDurationMs: host.intrinsicDurationMs,
-        };
-        throw new Error(
-          `Segment ID is not available for audio. Debug info: ${JSON.stringify(debugInfo)}`,
-        );
-      }
-
       return mediaEngine.fetchMediaSegment(segmentId, audioRendition, signal);
     },
   });

package/src/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.ts

@@ -14,9 +14,11 @@ export const makeAudioSegmentIdTask = (
     onComplete: (_value) => {},
     task: async ([, targetSeekTimeMs], { signal }) => {
       const mediaEngine = await getLatestMediaEngine(host, signal);
-      signal.throwIfAborted(); // Abort if a new seek started
+      signal.throwIfAborted();
 
-      const audioRendition = mediaEngine.audioRendition;
+      const audioRendition = mediaEngine.getAudioRendition();
+
+      // Return undefined if no audio rendition available (video-only asset)
       if (!audioRendition) {
         return undefined;
       }

package/src/elements/EFMedia/audioTasks/makeAudioTasksVideoOnly.browsertest.ts

@@ -0,0 +1,95 @@
+import { describe } from "vitest";
+import { test as baseTest } from "../../../../test/useMSW.js";
+import type { EFMedia } from "../../EFMedia.js";
+import { AssetMediaEngine } from "../AssetMediaEngine.js";
+
+const test = baseTest.extend<{
+  videoOnlyAssetEngine: AssetMediaEngine;
+}>({
+  videoOnlyAssetEngine: async ({}, use) => {
+    const host = document.createElement("ef-video") as EFMedia;
+    const engine = new AssetMediaEngine(host, "test-video-only.mp4");
+
+    // Simulate video-only asset data (no audio track) - this is the exact scenario
+    // that caused "computeSegmentId: trackId not found for rendition {\"src\":\"uuid\"}"
+    (engine as any).data = {
+      1: {
+        track: 1,
+        type: "video",
+        width: 480,
+        height: 270,
+        timescale: 15360,
+        sample_count: 1,
+        codec: "avc1.640015",
+        duration: 30208,
+        startTimeOffsetMs: 67,
+        initSegment: { offset: 0, size: 763 },
+        segments: [
+          { cts: 1024, dts: 0, duration: 30720, offset: 763, size: 13997 },
+        ],
+      },
+      // Note: No track 2 (audio) - this simulates the exact video-only asset scenario
+    };
+
+    await use(engine);
+  },
+});
+
+/**
+ * Regression test for: "computeSegmentId: trackId not found for rendition {\"src\":\"uuid\"}"
+ *
+ * This test ensures that AssetMediaEngine properly handles video-only assets
+ * by returning undefined for audio renditions instead of malformed objects.
+ *
+ * This test would FAIL with the old implementation and PASS with the new implementation.
+ */
+describe("AssetMediaEngine - Video-Only Asset Handling", () => {
+  test("audioRendition returns undefined for video-only asset", ({
+    videoOnlyAssetEngine,
+    expect,
+  }) => {
+    // This is the core fix - should return undefined, not {src: "..."}
+    const audioRendition = videoOnlyAssetEngine.audioRendition;
+    expect(audioRendition).toBeUndefined();
+  });
+
+  test("videoRendition returns valid object for video-only asset", ({
+    videoOnlyAssetEngine,
+    expect,
+  }) => {
+    const videoRendition = videoOnlyAssetEngine.videoRendition;
+    expect(videoRendition).toBeDefined();
+    expect(videoRendition?.trackId).toBe(1);
+    expect(videoRendition?.src).toBe("test-video-only.mp4");
+  });
+
+  test("getAudioRendition returns undefined for video-only asset", ({
+    videoOnlyAssetEngine,
+    expect,
+  }) => {
+    // New API behavior - should return undefined gracefully
+    const result = videoOnlyAssetEngine.getAudioRendition();
+    expect(result).toBeUndefined();
+  });
+
+  test("original error scenario is prevented", ({
+    videoOnlyAssetEngine,
+    expect,
+  }) => {
+    // This is the exact scenario that caused the original error:
+    // "computeSegmentId: trackId not found for rendition {\"src\":\"uuid\"}"
+
+    const audioRendition = videoOnlyAssetEngine.getAudioRendition();
+
+    // Before fix: audioRendition would be {trackId: undefined, src: "..."}
+    // After fix: audioRendition should be undefined
+    expect(audioRendition).toBeUndefined();
+
+    // This prevents the downstream error where trackId was missing entirely
+    if (audioRendition !== undefined) {
+      // If audioRendition exists, it should have a valid trackId
+      expect(audioRendition.trackId).toBeDefined();
+      expect(typeof audioRendition.trackId).toBe("number");
+    }
+  });
+});

package/src/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.ts

@@ -2,7 +2,7 @@ import { Task } from "@lit/task";
 
 import { EF_INTERACTIVE } from "../../../EF_INTERACTIVE.js";
 import { LRUCache } from "../../../utils/LRUCache.js";
-import type { EFMedia } from "../../EFMedia.js";
+import { type EFMedia, IgnorableError } from "../../EFMedia.js";
 
 // DECAY_WEIGHT constant - same as original
 const DECAY_WEIGHT = 0.8;
@@ -14,6 +14,10 @@ export function makeAudioTimeDomainAnalysisTask(element: EFMedia) {
   return new Task(element, {
     autoRun: EF_INTERACTIVE,
     onError: (error) => {
+      if (error instanceof IgnorableError) {
+        console.info("byteTimeDomainTask skipped: no audio track");
+        return;
+      }
       console.error("byteTimeDomainTask error", error);
     },
     args: () =>
@@ -27,11 +31,6 @@ export function makeAudioTimeDomainAnalysisTask(element: EFMedia) {
     task: async (_, { signal }) => {
       if (element.currentSourceTimeMs < 0) return null;
 
-      const mediaEngine = element.mediaEngineTask.value;
-      if (!mediaEngine?.audioRendition) {
-        return null;
-      }
-
       const currentTimeMs = element.currentSourceTimeMs;
 
       // Calculate exact audio window needed based on fftDecay and frame timing