@editframe/elements 0.18.23-beta.0 → 0.18.27-beta.0

package/src/elements/EFMedia/videoTasks/makeScrubVideoBufferTask.ts

@@ -0,0 +1,113 @@
+import { Task } from "@lit/task";
+import { EF_RENDERING } from "../../../EF_RENDERING.js";
+import type { VideoRendition } from "../../../transcoding/types";
+import type { EFVideo } from "../../EFVideo";
+import type { MediaBufferState } from "../shared/BufferUtils";
+import { manageMediaBuffer } from "../shared/BufferUtils";
+
+/**
+ * Scrub video buffer task - aggressively preloads the ENTIRE scrub track
+ * Unlike main video buffering, this loads the full duration with higher concurrency
+ * for instant visual feedback during seeking
+ */
+export const makeScrubVideoBufferTask = (host: EFVideo) => {
+  let currentState: MediaBufferState = {
+    currentSeekTimeMs: 0,
+    requestedSegments: new Set(),
+    activeRequests: new Set(),
+    requestQueue: [],
+  };
+
+  return new Task(host, {
+    // Run immediately in interactive mode for instant preloading
+    autoRun: !EF_RENDERING(),
+    args: () => [host.mediaEngineTask.value] as const,
+    onError: (error) => {
+      console.error("scrubVideoBufferTask error", error);
+    },
+    onComplete: (value) => {
+      currentState = value as MediaBufferState;
+    },
+    task: async ([mediaEngine], { signal }) => {
+      // Skip entirely in rendering mode
+      if (EF_RENDERING()) {
+        return currentState;
+      }
+
+      // Don't run if video buffering is disabled
+      if (!host.enableVideoBuffering) {
+        return currentState;
+      }
+
+      // Need media engine to be available
+      if (!mediaEngine) {
+        return currentState;
+      }
+
+      // Get scrub rendition using the proper interface method
+      const scrubRendition = mediaEngine.getScrubVideoRendition();
+
+      if (!scrubRendition) {
+        // No scrub rendition available - this is fine, just return current state
+        return currentState;
+      }
+
+      // Add src to rendition
+      const scrubRenditionWithSrc = {
+        ...scrubRendition,
+        src: mediaEngine.src,
+      };
+
+      try {
+        // First, ensure the init segment is cached for scrub content
+        try {
+          await mediaEngine.fetchInitSegment(scrubRenditionWithSrc, signal);
+        } catch (error) {
+          console.warn(
+            "ScrubBuffer: Failed to cache scrub init segment:",
+            error,
+          );
+        }
+
+        // Aggressively buffer the ENTIRE scrub track
+        const newState = await manageMediaBuffer<VideoRendition>(
+          0, // Always start from beginning to ensure complete coverage
+          {
+            // Buffer entire duration - scrub segments are 30s so much fewer total segments
+            bufferDurationMs: mediaEngine.durationMs,
+            // High concurrency for scrub - fewer segments overall (30s each vs 2s for main)
+            // E.g., 10min video = ~20 scrub segments vs ~300 main segments
+            maxParallelFetches: 10,
+            enableBuffering: true,
+            enableContinuousBuffering: true, // Keep going until all segments loaded
+          },
+          currentState,
+          mediaEngine.durationMs,
+          signal,
+          {
+            computeSegmentId: async (timeMs, rendition) => {
+              return mediaEngine.computeSegmentId(timeMs, rendition);
+            },
+            prefetchSegment: async (segmentId, rendition) => {
+              // Use the media engine to fetch and cache scrub segments
+              await mediaEngine.fetchMediaSegment(segmentId, rendition);
+            },
+            isSegmentCached: (segmentId, rendition) => {
+              return mediaEngine.isSegmentCached(segmentId, rendition);
+            },
+            getRendition: async () => scrubRenditionWithSrc,
+            logError: (message, error) => {
+              console.warn(`ScrubBuffer: ${message}`, error);
+            },
+          },
+        );
+
+        return newState;
+      } catch (error) {
+        if (signal.aborted) return currentState;
+        console.warn("ScrubBuffer failed:", error);
+        return currentState;
+      }
+    },
+  });
+};

package/src/elements/EFMedia/videoTasks/{makeVideoInitSegmentFetchTask.ts → makeScrubVideoInitSegmentFetchTask.ts}

@@ -3,19 +3,31 @@ import type { MediaEngine } from "../../../transcoding/types";
 import type { EFVideo } from "../../EFVideo";
 import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
 
-export const makeVideoInitSegmentFetchTask = (
+export const makeScrubVideoInitSegmentFetchTask = (
   host: EFVideo,
 ): Task<readonly [MediaEngine | undefined], ArrayBuffer> => {
   return new Task(host, {
     args: () => [host.mediaEngineTask.value] as const,
     onError: (error) => {
-      console.error("videoInitSegmentFetchTask error", error);
+      console.error("scrubVideoInitSegmentFetchTask error", error);
     },
     onComplete: (_value) => {},
     task: async ([_mediaEngine], { signal }) => {
       const mediaEngine = await getLatestMediaEngine(host, signal);
+
+      // Get scrub rendition using the proper interface method
+      const scrubRendition = mediaEngine.getScrubVideoRendition();
+
+      if (!scrubRendition) {
+        // No scrub rendition available - this is fine, scrub is optional
+        throw new Error("No scrub rendition available");
+      }
+
       return mediaEngine.fetchInitSegment(
-        mediaEngine.getVideoRendition(),
+        {
+          ...scrubRendition,
+          src: mediaEngine.src, // Ensure src is set
+        },
         signal,
       );
     },

package/src/elements/EFMedia/videoTasks/{makeVideoInputTask.ts → makeScrubVideoInputTask.ts}

@@ -5,31 +5,32 @@ import type { EFVideo } from "../../EFVideo";
 import { BufferedSeekingInput } from "../BufferedSeekingInput";
 import type { InputTask } from "../shared/MediaTaskUtils";
 
-export const makeVideoInputTask = (host: EFVideo): InputTask => {
+export const makeScrubVideoInputTask = (host: EFVideo): InputTask => {
   return new Task<
     readonly [ArrayBuffer | undefined, ArrayBuffer | undefined],
     BufferedSeekingInput
   >(host, {
     args: () =>
       [
-        host.videoInitSegmentFetchTask.value,
-        host.videoSegmentFetchTask.value,
+        host.scrubVideoInitSegmentFetchTask.value,
+        host.scrubVideoSegmentFetchTask.value,
       ] as const,
     onError: (error) => {
-      console.error("videoInputTask error", error);
+      console.error("scrubVideoInputTask error", error);
     },
     onComplete: (_value) => {},
     task: async () => {
-      const initSegment = await host.videoInitSegmentFetchTask.taskComplete;
-      const segment = await host.videoSegmentFetchTask.taskComplete;
+      const initSegment =
+        await host.scrubVideoInitSegmentFetchTask.taskComplete;
+      const segment = await host.scrubVideoSegmentFetchTask.taskComplete;
       if (!initSegment || !segment) {
-        throw new Error("Init segment or segment is not available");
+        throw new Error("Scrub init segment or segment is not available");
       }
 
-      // Get startTimeOffsetMs from the video rendition if available
+      // Get startTimeOffsetMs from the scrub rendition if available
       const mediaEngine = await host.mediaEngineTask.taskComplete;
-      const videoRendition = mediaEngine?.videoRendition;
-      const startTimeOffsetMs = videoRendition?.startTimeOffsetMs;
+      const scrubRendition = mediaEngine.getScrubVideoRendition();
+      const startTimeOffsetMs = scrubRendition?.startTimeOffsetMs;
 
       const input = new BufferedSeekingInput(
         await new Blob([initSegment, segment]).arrayBuffer(),

package/src/elements/EFMedia/videoTasks/makeScrubVideoSeekTask.ts

@@ -0,0 +1,118 @@
+import { Task } from "@lit/task";
+
+import type { VideoSample } from "mediabunny";
+
+import type { EFVideo } from "../../EFVideo";
+import { ScrubInputCache } from "./ScrubInputCache";
+
+type ScrubVideoSeekTask = Task<readonly [number], VideoSample | undefined>;
+
+// Shared cache instance across all scrub seek tasks
+const scrubInputCache = new ScrubInputCache();
+
+export const makeScrubVideoSeekTask = (host: EFVideo): ScrubVideoSeekTask => {
+  return new Task(host, {
+    args: () => [host.desiredSeekTimeMs] as const,
+    onError: (error) => {
+      console.error("scrubVideoSeekTask error", error);
+    },
+    onComplete: (_value) => {},
+    task: async ([desiredSeekTimeMs], { signal }) => {
+      signal.throwIfAborted();
+
+      const mediaEngine = host.mediaEngineTask.value;
+      if (!mediaEngine) {
+        return undefined;
+      }
+
+      // Get scrub rendition using the proper interface method
+      const scrubRendition = mediaEngine.getScrubVideoRendition();
+
+      if (!scrubRendition) {
+        return undefined; // No scrub rendition
+      }
+
+      const scrubRenditionWithSrc = {
+        ...scrubRendition,
+        src: mediaEngine.src,
+      };
+
+      // Compute which scrub segment (30s) contains the desired seek time
+      const segmentId = mediaEngine.computeSegmentId(
+        desiredSeekTimeMs,
+        scrubRenditionWithSrc,
+      );
+      if (segmentId === undefined) {
+        return undefined;
+      }
+
+      // Check if this scrub segment is cached (preloaded by scrub buffer task)
+      const isCached = mediaEngine.isSegmentCached(
+        segmentId,
+        scrubRenditionWithSrc,
+      );
+      if (!isCached) {
+        // Scrub content not preloaded - fail fast, let main video handle it
+        // Scrub's job is instant feedback from cached content only
+        return undefined;
+      }
+
+      signal.throwIfAborted();
+
+      try {
+        // Get or create BufferedSeekingInput for this scrub segment (30s)
+        // This efficiently reuses the same input for seeks within the same 30s range
+        const scrubInput = await scrubInputCache.getOrCreateInput(
+          segmentId,
+          async () => {
+            const [initSegment, mediaSegment] = await Promise.all([
+              mediaEngine.fetchInitSegment(scrubRenditionWithSrc, signal),
+              mediaEngine.fetchMediaSegment(segmentId, scrubRenditionWithSrc),
+            ]);
+
+            if (!initSegment || !mediaSegment || signal.aborted) {
+              return undefined;
+            }
+
+            const { BufferedSeekingInput } = await import(
+              "../BufferedSeekingInput.js"
+            );
+            const { EFMedia } = await import("../../EFMedia.js");
+
+            return new BufferedSeekingInput(
+              await new Blob([initSegment, mediaSegment]).arrayBuffer(),
+              {
+                videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
+                audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
+                startTimeOffsetMs: scrubRendition.startTimeOffsetMs,
+              },
+            );
+          },
+        );
+
+        if (!scrubInput) {
+          return undefined;
+        }
+
+        // Get video track and seek to precise time within the 30s scrub segment
+        const videoTrack = await scrubInput.getFirstVideoTrack();
+        if (!videoTrack) {
+          return undefined;
+        }
+
+        signal.throwIfAborted();
+
+        const sample = (await scrubInput.seek(
+          videoTrack.id,
+          desiredSeekTimeMs,
+        )) as unknown as VideoSample | undefined;
+
+        return sample;
+      } catch (error) {
+        if (signal.aborted) return undefined;
+        console.warn("Failed to get scrub video sample:", error);
+        return undefined;
+      }
+    },
+  });
+};

package/src/elements/EFMedia/videoTasks/makeScrubVideoSegmentFetchTask.ts

@@ -0,0 +1,44 @@
+import { Task } from "@lit/task";
+import type { MediaEngine } from "../../../transcoding/types";
+import type { EFVideo } from "../../EFVideo";
+import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
+
+export const makeScrubVideoSegmentFetchTask = (
+  host: EFVideo,
+): Task<
+  readonly [MediaEngine | undefined, number | undefined],
+  ArrayBuffer
+> => {
+  return new Task(host, {
+    args: () =>
+      [host.mediaEngineTask.value, host.scrubVideoSegmentIdTask.value] as const,
+    onError: (error) => {
+      console.error("scrubVideoSegmentFetchTask error", error);
+    },
+    onComplete: (_value) => {},
+    task: async (_, { signal }) => {
+      const mediaEngine = await getLatestMediaEngine(host, signal);
+      const segmentId = await host.scrubVideoSegmentIdTask.taskComplete;
+      if (segmentId === undefined) {
+        throw new Error("Scrub segment ID is not available for video");
+      }
+
+      // Get scrub rendition using the proper interface method
+      const scrubRendition = mediaEngine.getScrubVideoRendition();
+
+      if (!scrubRendition) {
+        // No scrub rendition available - this is fine, scrub is optional
+        throw new Error("No scrub rendition available");
+      }
+
+      return mediaEngine.fetchMediaSegment(
+        segmentId,
+        {
+          ...scrubRendition,
+          src: mediaEngine.src, // Ensure src is set
+        },
+        signal,
+      );
+    },
+  });
+};

package/src/elements/EFMedia/videoTasks/{makeVideoSegmentIdTask.ts → makeScrubVideoSegmentIdTask.ts}

@@ -3,22 +3,30 @@ import type { MediaEngine } from "../../../transcoding/types";
 import type { EFVideo } from "../../EFVideo";
 import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
 
-export const makeVideoSegmentIdTask = (
+export const makeScrubVideoSegmentIdTask = (
   host: EFVideo,
 ): Task<readonly [MediaEngine | undefined, number], number | undefined> => {
   return new Task(host, {
     args: () => [host.mediaEngineTask.value, host.desiredSeekTimeMs] as const,
     onError: (error) => {
-      console.error("videoSegmentIdTask error", error);
+      console.error("scrubVideoSegmentIdTask error", error);
     },
     onComplete: (_value) => {},
     task: async ([, targetSeekTimeMs], { signal }) => {
       const mediaEngine = await getLatestMediaEngine(host, signal);
       signal.throwIfAborted(); // Abort if a new seek started
-      return mediaEngine.computeSegmentId(
-        targetSeekTimeMs, // Use captured value, not host.desiredSeekTimeMs
-        mediaEngine.getVideoRendition(),
-      );
+
+      // Get scrub rendition using the proper interface method
+      const scrubRendition = mediaEngine.getScrubVideoRendition();
+
+      if (!scrubRendition) {
+        return undefined; // No scrub rendition available
+      }
+
+      return mediaEngine.computeSegmentId(targetSeekTimeMs, {
+        ...scrubRendition,
+        src: mediaEngine.src, // Ensure src is set
+      });
     },
   });
 };

package/src/elements/EFMedia/videoTasks/makeUnifiedVideoSeekTask.ts

@@ -0,0 +1,258 @@
+import { Task } from "@lit/task";
+import type { VideoSample } from "mediabunny";
+import type { VideoRendition } from "../../../transcoding/types";
+import type { EFVideo } from "../../EFVideo";
+import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
+import { ScrubInputCache } from "./ScrubInputCache";
+
+type UnifiedVideoSeekTask = Task<readonly [number], VideoSample | undefined>;
+
+// Shared cache for scrub inputs
+const scrubInputCache = new ScrubInputCache();
+
+export const makeUnifiedVideoSeekTask = (
+  host: EFVideo,
+): UnifiedVideoSeekTask => {
+  return new Task(host, {
+    args: () => [host.desiredSeekTimeMs] as const,
+    onError: (error) => {
+      console.error("unifiedVideoSeekTask error", error);
+    },
+    onComplete: (_value) => {},
+    task: async ([desiredSeekTimeMs], { signal }) => {
+      signal.throwIfAborted();
+
+      const mediaEngine = await getLatestMediaEngine(host, signal);
+      if (!mediaEngine) return undefined;
+
+      // FIRST: Check if main quality content is already cached
+      const mainRendition = mediaEngine.videoRendition;
+      if (mainRendition) {
+        const mainSegmentId = mediaEngine.computeSegmentId(
+          desiredSeekTimeMs,
+          mainRendition,
+        );
+        if (
+          mainSegmentId !== undefined &&
+          mediaEngine.isSegmentCached(mainSegmentId, mainRendition)
+        ) {
+          // Main content is cached! Use it directly.
+          return await getMainVideoSample(
+            host,
+            mediaEngine,
+            desiredSeekTimeMs,
+            signal,
+          );
+        }
+      }
+
+      // SECOND: Main not cached, try scrub path (instant if cached)
+      const scrubSample = await tryGetScrubSample(
+        mediaEngine,
+        desiredSeekTimeMs,
+        signal,
+      );
+      if (scrubSample || signal.aborted) {
+        // If scrub succeeded, start background main quality upgrade (non-blocking)
+        if (scrubSample) {
+          startMainQualityUpgrade(
+            host,
+            mediaEngine,
+            desiredSeekTimeMs,
+            signal,
+          ).catch(() => {
+            // Main upgrade failed - scrub already succeeded, that's fine
+          });
+        }
+
+        return scrubSample;
+      }
+
+      // THIRD: Neither are cached, fetch main video path as final fallback
+      return await getMainVideoSample(
+        host,
+        mediaEngine,
+        desiredSeekTimeMs,
+        signal,
+      );
+    },
+  });
+};
+
+/**
+ * Try to get scrub sample from cache (instant if available)
+ */
+async function tryGetScrubSample(
+  mediaEngine: any,
+  desiredSeekTimeMs: number,
+  signal: AbortSignal,
+): Promise<VideoSample | undefined> {
+  try {
+    // Get scrub rendition
+    let scrubRendition: VideoRendition | undefined;
+    if ("data" in mediaEngine && mediaEngine.data?.videoRenditions) {
+      scrubRendition = mediaEngine.data.videoRenditions.find(
+        (r: any) => r.id === "scrub",
+      );
+    }
+
+    if (!scrubRendition) return undefined;
+
+    const scrubRenditionWithSrc = {
+      ...scrubRendition,
+      src: mediaEngine.src,
+    };
+
+    // Check if scrub segment is cached
+    const segmentId = mediaEngine.computeSegmentId(
+      desiredSeekTimeMs,
+      scrubRenditionWithSrc,
+    );
+    if (segmentId === undefined) return undefined;
+
+    const isCached = mediaEngine.isSegmentCached(
+      segmentId,
+      scrubRenditionWithSrc,
+    );
+    if (!isCached) return undefined; // Not cached - let main video handle it
+
+    // Get cached scrub input and seek within it
+    const scrubInput = await scrubInputCache.getOrCreateInput(
+      segmentId,
+      async () => {
+        const [initSegment, mediaSegment] = await Promise.all([
+          mediaEngine.fetchInitSegment(scrubRenditionWithSrc, signal),
+          mediaEngine.fetchMediaSegment(segmentId, scrubRenditionWithSrc),
+        ]);
+
+        if (!initSegment || !mediaSegment || signal.aborted) return undefined;
+
+        const { BufferedSeekingInput } = await import(
+          "../BufferedSeekingInput.js"
+        );
+        const { EFMedia } = await import("../../EFMedia.js");
+
+        return new BufferedSeekingInput(
+          await new Blob([initSegment, mediaSegment]).arrayBuffer(),
+          {
+            videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
+            audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
+            startTimeOffsetMs: scrubRendition.startTimeOffsetMs,
+          },
+        );
+      },
+    );
+
+    if (!scrubInput) return undefined;
+
+    const videoTrack = await scrubInput.getFirstVideoTrack();
+    if (!videoTrack) return undefined;
+
+    const sample = (await scrubInput.seek(
+      videoTrack.id,
+      desiredSeekTimeMs,
+    )) as unknown as VideoSample | undefined;
+
+    return sample;
+  } catch (_error) {
+    if (signal.aborted) return undefined;
+    return undefined; // Scrub failed - let main video handle it
+  }
+}
+
+/**
+ * Get main video sample (slower path with fetching)
+ */
+async function getMainVideoSample(
+  _host: EFVideo,
+  mediaEngine: any,
+  desiredSeekTimeMs: number,
+  signal: AbortSignal,
+): Promise<VideoSample | undefined> {
+  try {
+    // Use existing main video task chain
+    const segmentId = mediaEngine.computeSegmentId(
+      desiredSeekTimeMs,
+      mediaEngine.getVideoRendition(),
+    );
+    if (segmentId === undefined) return undefined;
+
+    // Fetch main video segment
+    const [initSegment, mediaSegment] = await Promise.all([
+      mediaEngine.fetchInitSegment(mediaEngine.getVideoRendition(), signal),
+      mediaEngine.fetchMediaSegment(
+        segmentId,
+        mediaEngine.getVideoRendition(),
+        signal,
+      ),
+    ]);
+
+    if (!initSegment || !mediaSegment) return undefined;
+    signal.throwIfAborted();
+
+    // Create main video input
+    const { BufferedSeekingInput } = await import("../BufferedSeekingInput.js");
+    const { EFMedia } = await import("../../EFMedia.js");
+
+    const videoRendition = mediaEngine.videoRendition;
+    const startTimeOffsetMs = videoRendition?.startTimeOffsetMs;
+
+    const mainInput = new BufferedSeekingInput(
+      await new Blob([initSegment, mediaSegment]).arrayBuffer(),
+      {
+        videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
+        audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
+        startTimeOffsetMs,
+      },
+    );
+
+    const videoTrack = await mainInput.getFirstVideoTrack();
+    if (!videoTrack) return undefined;
+
+    signal.throwIfAborted();
+
+    const sample = (await mainInput.seek(
+      videoTrack.id,
+      desiredSeekTimeMs,
+    )) as unknown as VideoSample | undefined;
+
+    return sample;
+  } catch (error) {
+    if (signal.aborted) return undefined;
+    throw error;
+  }
+}
+
+/**
+ * Start background upgrade to main quality (non-blocking)
+ */
+async function startMainQualityUpgrade(
+  host: EFVideo,
+  mediaEngine: any,
+  targetSeekTimeMs: number,
+  signal: AbortSignal,
+): Promise<void> {
+  // Small delay to let scrub content display first
+  await new Promise((resolve) => setTimeout(resolve, 50));
+  if (signal.aborted || host.desiredSeekTimeMs !== targetSeekTimeMs) return;
+
+  // Get main quality sample and upgrade display
+  const mainSample = await getMainVideoSample(
+    host,
+    mediaEngine,
+    targetSeekTimeMs,
+    signal,
+  );
+  if (
+    mainSample &&
+    !signal.aborted &&
+    host.desiredSeekTimeMs === targetSeekTimeMs
+  ) {
+    const videoFrame = mainSample.toVideoFrame();
+    try {
+      host.displayFrame(videoFrame, targetSeekTimeMs);
+    } finally {
+      videoFrame.close();
+    }
+  }
+}