@editframe/elements 0.19.4-beta.0 → 0.20.0-beta.0

package/src/elements/EFMedia/BaseMediaEngine.browsertest.ts

@@ -7,6 +7,7 @@ import { BaseMediaEngine, mediaCache } from "./BaseMediaEngine.js";
 const test = baseTest.extend<{}>({});
 
 // Test implementation of BaseMediaEngine for testing
+// @ts-expect-error missing implementations
 class TestMediaEngine extends BaseMediaEngine {
   fetchMediaSegment = vi.fn();
   public host: EFMedia;
@@ -221,14 +222,8 @@ describe("BaseMediaEngine abort signal handling", () => {
     const host = {
       fetch: vi
         .fn()
-        .mockImplementation(
-          () =>
-            new Promise((resolve) =>
-              setTimeout(
-                () => resolve({ arrayBuffer: () => new ArrayBuffer(1024) }),
-                100,
-              ),
-            ),
+        .mockImplementation(() =>
+          Promise.resolve({ arrayBuffer: () => new ArrayBuffer(1024) }),
         ),
     } as any;
 

package/src/elements/EFMedia/BaseMediaEngine.ts

@@ -2,10 +2,12 @@ import { RequestDeduplicator } from "../../transcoding/cache/RequestDeduplicator
 import type {
   AudioRendition,
   SegmentTimeRange,
+  ThumbnailResult,
   VideoRendition,
 } from "../../transcoding/types";
 import { SizeAwareLRUCache } from "../../utils/LRUCache.js";
 import type { EFMedia } from "../EFMedia.js";
+import type { MediaRendition } from "./shared/MediaTaskUtils.js";
 
 // Global instances shared across all media engines
 export const mediaCache = new SizeAwareLRUCache<string>(100 * 1024 * 1024); // 100MB cache limit
@@ -205,6 +207,16 @@ export abstract class BaseMediaEngine {
     rendition: { trackId: number | undefined; src: string },
   ): Promise<ArrayBuffer>;
 
+  abstract fetchInitSegment(
+    rendition: { trackId: number | undefined; src: string },
+    signal: AbortSignal,
+  ): Promise<ArrayBuffer>;
+
+  abstract computeSegmentId(
+    desiredSeekTimeMs: number,
+    rendition: MediaRendition,
+  ): number | undefined;
+
   /**
    * Fetch media segment with built-in deduplication
    * Now uses global deduplication for all requests
@@ -387,4 +399,27 @@ export abstract class BaseMediaEngine {
       segmentIds.filter((id) => this.isSegmentCached(id, rendition)),
     );
   }
+
+  /**
+   * Extract thumbnail canvases at multiple timestamps efficiently
+   * Default implementation provides helpful error information
+   */
+  async extractThumbnails(
+    timestamps: number[],
+  ): Promise<(ThumbnailResult | null)[]> {
+    const engineName = this.constructor.name;
+    console.warn(
+      `${engineName}: extractThumbnails not properly implemented. ` +
+        "This MediaEngine type does not support thumbnail generation. " +
+        "Supported engines: JitMediaEngine. " +
+        `Requested ${timestamps.length} thumbnail${timestamps.length === 1 ? "" : "s"}.`,
+    );
+    return timestamps.map(() => null);
+  }
+
+  abstract convertToSegmentRelativeTimestamps(
+    globalTimestamps: number[],
+    segmentId: number,
+    rendition: VideoRendition,
+  ): number[];
 }

package/src/elements/EFMedia/JitMediaEngine.ts

@@ -2,16 +2,19 @@ import type {
   AudioRendition,
   MediaEngine,
   RenditionId,
+  ThumbnailResult,
   VideoRendition,
 } from "../../transcoding/types";
 import type { ManifestResponse } from "../../transcoding/types/index.js";
 import type { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
 import type { EFMedia } from "../EFMedia.js";
 import { BaseMediaEngine } from "./BaseMediaEngine";
+import { ThumbnailExtractor } from "./shared/ThumbnailExtractor.js";
 
 export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
   private urlGenerator: UrlGenerator;
   private data: ManifestResponse = {} as ManifestResponse;
+  private thumbnailExtractor: ThumbnailExtractor;
 
   static async fetch(host: EFMedia, urlGenerator: UrlGenerator, url: string) {
     const engine = new JitMediaEngine(host, urlGenerator);
@@ -23,6 +26,7 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
   constructor(host: EFMedia, urlGenerator: UrlGenerator) {
     super(host);
     this.urlGenerator = urlGenerator;
+    this.thumbnailExtractor = new ThumbnailExtractor(this);
   }
 
   get durationMs() {
@@ -204,4 +208,48 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
       maxAudioBufferFetches: 3,
     };
   }
+
+  /**
+   * Extract thumbnail canvases using same rendition priority as video playback for frame alignment
+   */
+  async extractThumbnails(
+    timestamps: number[],
+  ): Promise<(ThumbnailResult | null)[]> {
+    // Use same rendition priority as video: try main rendition first for frame alignment
+    let rendition: VideoRendition;
+    try {
+      const mainRendition = this.getVideoRendition();
+      if (mainRendition) {
+        rendition = mainRendition;
+      } else {
+        const scrubRendition = this.getScrubVideoRendition();
+        if (scrubRendition) {
+          rendition = scrubRendition;
+        } else {
+          throw new Error("No video rendition available");
+        }
+      }
+    } catch (error) {
+      console.warn(
+        "JitMediaEngine: No video rendition available for thumbnails",
+        error,
+      );
+      return timestamps.map(() => null);
+    }
+
+    // Use shared thumbnail extraction logic
+    return this.thumbnailExtractor.extractThumbnails(
+      timestamps,
+      rendition,
+      this.durationMs,
+    );
+  }
+
+  convertToSegmentRelativeTimestamps(
+    globalTimestamps: number[],
+    _segmentId: number,
+    _rendition: VideoRendition,
+  ): number[] {
+    return globalTimestamps.map((timestamp) => timestamp / 1000);
+  }
 }

package/src/elements/EFMedia/shared/GlobalInputCache.ts

@@ -0,0 +1,77 @@
+import type { Input } from "mediabunny";
+import { LRUCache } from "../../../utils/LRUCache.js";
+
+/**
+ * Global cache for MediaBunny Input instances
+ * Shared across all MediaEngine instances to prevent duplicate decoding
+ * of the same segment data
+ */
+class GlobalInputCache {
+  private cache = new LRUCache<string, Input>(50); // 50 Input instances max
+
+  /**
+   * Generate standardized cache key for Input objects
+   * Format: "input:{src}:{segmentId}:{renditionId}"
+   */
+  private generateKey(
+    src: string,
+    segmentId: number,
+    renditionId?: string,
+  ): string {
+    return `input:${src}:${segmentId}:${renditionId || "default"}`;
+  }
+
+  /**
+   * Get cached Input object
+   */
+  get(src: string, segmentId: number, renditionId?: string): Input | undefined {
+    const key = this.generateKey(src, segmentId, renditionId);
+    return this.cache.get(key);
+  }
+
+  /**
+   * Cache Input object
+   */
+  set(
+    src: string,
+    segmentId: number,
+    input: Input,
+    renditionId?: string,
+  ): void {
+    const key = this.generateKey(src, segmentId, renditionId);
+    this.cache.set(key, input);
+  }
+
+  /**
+   * Check if Input is cached
+   */
+  has(src: string, segmentId: number, renditionId?: string): boolean {
+    const key = this.generateKey(src, segmentId, renditionId);
+    return this.cache.has(key);
+  }
+
+  /**
+   * Clear all cached Input objects
+   */
+  clear(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * Get cache statistics for debugging
+   */
+  getStats() {
+    return {
+      size: this.cache.size,
+      cachedKeys: Array.from((this.cache as any).cache.keys()),
+    };
+  }
+}
+
+// Single global instance shared across all MediaEngine instances
+export const globalInputCache = new GlobalInputCache();
+
+// Export for debugging (works in both browser and server)
+(
+  globalThis as typeof globalThis & { debugInputCache: typeof globalInputCache }
+).debugInputCache = globalInputCache;

package/src/elements/EFMedia/shared/ThumbnailExtractor.ts

@@ -0,0 +1,227 @@
+import { ALL_FORMATS, BlobSource, CanvasSink, Input } from "mediabunny";
+import type {
+  ThumbnailResult,
+  VideoRendition,
+} from "../../../transcoding/types/index.js";
+import type { BaseMediaEngine } from "../BaseMediaEngine.js";
+import { globalInputCache } from "./GlobalInputCache.js";
+
+/**
+ * Shared thumbnail extraction logic for all MediaEngine implementations
+ * Eliminates code duplication and provides consistent behavior
+ */
+export class ThumbnailExtractor {
+  constructor(private mediaEngine: BaseMediaEngine) {}
+
+  /**
+   * Extract thumbnails at multiple timestamps efficiently using segment batching
+   */
+  async extractThumbnails(
+    timestamps: number[],
+    rendition: VideoRendition,
+    durationMs: number,
+  ): Promise<(ThumbnailResult | null)[]> {
+    if (timestamps.length === 0) {
+      return [];
+    }
+
+    // Validate and filter timestamps within bounds
+    const validTimestamps = timestamps.filter(
+      (timeMs) => timeMs >= 0 && timeMs <= durationMs,
+    );
+
+    if (validTimestamps.length === 0) {
+      console.warn(
+        `ThumbnailExtractor: All timestamps out of bounds (0-${durationMs}ms)`,
+      );
+      return timestamps.map(() => null);
+    }
+
+    // Group timestamps by segment for batch processing
+    const segmentGroups = this.groupTimestampsBySegment(
+      validTimestamps,
+      rendition,
+    );
+
+    // Extract batched by segment using CanvasSink
+    const results = new Map<number, ThumbnailResult | null>();
+
+    for (const [segmentId, segmentTimestamps] of segmentGroups) {
+      try {
+        const segmentResults = await this.extractSegmentThumbnails(
+          segmentId,
+          segmentTimestamps,
+          rendition,
+        );
+
+        for (const [timestamp, thumbnail] of segmentResults) {
+          results.set(timestamp, thumbnail);
+        }
+      } catch (error) {
+        console.warn(
+          `ThumbnailExtractor: Failed to extract thumbnails for segment ${segmentId}:`,
+          error,
+        );
+        // Mark all timestamps in this segment as failed
+        for (const timestamp of segmentTimestamps) {
+          results.set(timestamp, null);
+        }
+      }
+    }
+
+    // Return in original order, null for any that failed or were out of bounds
+    return timestamps.map((t) => {
+      // If timestamp was out of bounds, return null
+      if (t < 0 || t > durationMs) {
+        return null;
+      }
+      return results.get(t) || null;
+    });
+  }
+
+  /**
+   * Group timestamps by segment ID for efficient batch processing
+   */
+  private groupTimestampsBySegment(
+    timestamps: number[],
+    rendition: VideoRendition,
+  ): Map<number, number[]> {
+    const segmentGroups = new Map<number, number[]>();
+
+    for (const timeMs of timestamps) {
+      try {
+        const segmentId = this.mediaEngine.computeSegmentId(timeMs, rendition);
+        if (segmentId !== undefined) {
+          if (!segmentGroups.has(segmentId)) {
+            segmentGroups.set(segmentId, []);
+          }
+          const segmentGroup = segmentGroups.get(segmentId) ?? [];
+          if (!segmentGroup) {
+            segmentGroups.set(segmentId, []);
+          }
+          segmentGroup.push(timeMs);
+        }
+      } catch (error) {
+        console.warn(
+          `ThumbnailExtractor: Could not compute segment for timestamp ${timeMs}:`,
+          error,
+        );
+      }
+    }
+
+    return segmentGroups;
+  }
+
+  /**
+   * Extract thumbnails for a specific segment using CanvasSink
+   */
+  private async extractSegmentThumbnails(
+    segmentId: number,
+    timestamps: number[],
+    rendition: VideoRendition,
+  ): Promise<Map<number, ThumbnailResult | null>> {
+    const results = new Map<number, ThumbnailResult | null>();
+
+    try {
+      // Get segment data through existing media engine methods (uses caches)
+      const abortController = new AbortController();
+      const [initSegment, mediaSegment] = await Promise.all([
+        this.mediaEngine.fetchInitSegment(rendition, abortController.signal),
+        this.mediaEngine.fetchMediaSegment(segmentId, rendition),
+      ]);
+
+      // Create Input for this segment using global shared cache
+      const segmentBlob = new Blob([initSegment, mediaSegment]);
+
+      let input = globalInputCache.get(rendition.src, segmentId, rendition.id);
+      if (!input) {
+        input = new Input({
+          formats: ALL_FORMATS,
+          source: new BlobSource(segmentBlob),
+        });
+        globalInputCache.set(rendition.src, segmentId, input, rendition.id);
+      }
+
+      // Set up CanvasSink for batched extraction
+      const videoTrack = await input.getPrimaryVideoTrack();
+      if (!videoTrack) {
+        // No video track - return nulls for all timestamps
+        for (const timestamp of timestamps) {
+          results.set(timestamp, null);
+        }
+        return results;
+      }
+
+      const sink = new CanvasSink(videoTrack);
+
+      // Convert global timestamps to segment-relative (in seconds for mediabunny)
+      const relativeTimestamps = this.convertToSegmentRelativeTimestamps(
+        timestamps,
+        segmentId,
+        rendition,
+      );
+
+      // Batch extract all thumbnails for this segment
+      const timestampResults = [];
+      for await (const result of sink.canvasesAtTimestamps(
+        relativeTimestamps,
+      )) {
+        timestampResults.push(result);
+      }
+
+      // Map results back to original timestamps
+      for (let i = 0; i < timestamps.length; i++) {
+        const globalTimestamp = timestamps[i];
+        if (globalTimestamp === undefined) {
+          continue;
+        }
+
+        const result = timestampResults[i];
+
+        if (result?.canvas) {
+          const canvas = result.canvas;
+          if (
+            canvas instanceof HTMLCanvasElement ||
+            canvas instanceof OffscreenCanvas
+          ) {
+            results.set(globalTimestamp, {
+              timestamp: globalTimestamp,
+              thumbnail: canvas,
+            });
+          } else {
+            results.set(globalTimestamp, null);
+          }
+        } else {
+          results.set(globalTimestamp, null);
+        }
+      }
+    } catch (error) {
+      console.error(
+        `ThumbnailExtractor: Failed to extract thumbnails for segment ${segmentId}:`,
+        error,
+      );
+      // Return nulls for all timestamps on error
+      for (const timestamp of timestamps) {
+        results.set(timestamp, null);
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Convert global timestamps to segment-relative timestamps for mediabunny
+   * This is where the main difference between JIT and Asset engines lies
+   */
+  private convertToSegmentRelativeTimestamps(
+    globalTimestamps: number[],
+    segmentId: number,
+    rendition: VideoRendition,
+  ): number[] {
+    return this.mediaEngine.convertToSegmentRelativeTimestamps(
+      globalTimestamps,
+      segmentId,
+      rendition,
+    );
+  }
+}

package/src/elements/EFMedia.ts

@@ -1,6 +1,6 @@
 import { css, LitElement, type PropertyValueMap } from "lit";
 import { property, state } from "lit/decorators.js";
-
+import { isContextMixin } from "../gui/ContextMixin.js";
 import type { AudioSpan } from "../transcoding/types/index.ts";
 import { UrlGenerator } from "../transcoding/utils/UrlGenerator.ts";
 import { makeAudioBufferTask } from "./EFMedia/audioTasks/makeAudioBufferTask.ts";
@@ -64,6 +64,8 @@ export class EFMedia extends EFTargetable(
       "audio-buffer-duration",
       "max-audio-buffer-fetches",
       "enable-audio-buffering",
+      "sourcein",
+      "sourceout",
     ];
   }
 
@@ -212,6 +214,41 @@ export class EFMedia extends EFTargetable(
     if (changedProperties.has("ownCurrentTimeMs")) {
       this.executeSeek(this.currentSourceTimeMs);
     }
+
+    // Check if trim/source properties changed that affect duration
+    const durationAffectingProps = [
+      "_trimStartMs",
+      "_trimEndMs",
+      "_sourceInMs",
+      "_sourceOutMs",
+    ];
+
+    const hasDurationChange = durationAffectingProps.some((prop) =>
+      changedProperties.has(prop),
+    );
+
+    if (hasDurationChange) {
+      // Notify parent timegroup to recalculate its duration (same pattern as EFCaptions)
+      if (this.parentTimegroup) {
+        this.parentTimegroup.requestUpdate("durationMs");
+        this.parentTimegroup.requestUpdate("currentTime");
+
+        // Also find and directly notify any context provider (ContextMixin)
+        let parent = this.parentNode;
+        while (parent) {
+          if (isContextMixin(parent)) {
+            parent.dispatchEvent(
+              new CustomEvent("child-duration-changed", {
+                detail: { source: this },
+              }),
+            );
+            break;
+          }
+          parent = parent.parentNode;
+        }
+      }
+    }
+
     // if (
     //   changedProperties.has("currentTime") ||
     //   changedProperties.has("ownCurrentTimeMs")

package/src/elements/EFSurface.browsertest.ts

@@ -0,0 +1,155 @@
+import { html, render } from "lit";
+import { beforeEach, describe } from "vitest";
+
+import { test as baseTest } from "../../test/useMSW.js";
+
+import "./EFVideo.js";
+import "./EFTimegroup.js";
+import "../gui/EFPreview.js";
+import "../gui/EFWorkbench.js";
+import "./EFSurface.js";
+
+import type { EFSurface } from "./EFSurface.js";
+import type { EFTimegroup } from "./EFTimegroup.js";
+import type { EFVideo } from "./EFVideo.js";
+
+beforeEach(() => {
+  localStorage.clear();
+});
+
+const surfaceTest = baseTest.extend<{
+  timegroup: EFTimegroup;
+  video: EFVideo;
+  surface: EFSurface;
+}>({
+  timegroup: async ({}, use) => {
+    const container = document.createElement("div");
+    render(
+      html`
+        <ef-configuration api-host="http://localhost:63315">
+          <ef-preview>
+            <ef-timegroup id="tg" mode="sequence" class="relative h-[360px] w-[640px] overflow-hidden bg-black">
+              <ef-video id="vid" src="bars-n-tone.mp4" style="width: 100%; height: 100%;"></ef-video>
+              <ef-surface id="surf" target="vid" style="position: absolute; inset: 0;"></ef-surface>
+            </ef-timegroup>
+          </ef-preview>
+        </ef-configuration>
+      `,
+      container,
+    );
+    document.body.appendChild(container);
+    const configuration = container.querySelector("ef-configuration") as any;
+    configuration.signingURL = "";
+    const tg = container.querySelector("#tg") as EFTimegroup;
+    await tg.updateComplete;
+    await use(tg);
+    container.remove();
+  },
+  video: async ({ timegroup }, use) => {
+    const video = timegroup.querySelector("#vid") as EFVideo;
+    await video.updateComplete;
+    await use(video);
+  },
+  surface: async ({ timegroup }, use) => {
+    const surface = timegroup.querySelector("#surf") as unknown as EFSurface;
+    await surface.updateComplete;
+    await use(surface);
+  },
+});
+
+describe("EFSurface", () => {
+  surfaceTest("defines and renders a canvas", async ({ expect }) => {
+    const el = document.createElement("ef-surface");
+    document.body.appendChild(el);
+    await (el as any).updateComplete;
+    const canvas = el.shadowRoot?.querySelector("canvas");
+    expect(canvas).toBeTruthy();
+    expect((canvas as HTMLCanvasElement).tagName).toBe("CANVAS");
+    el.remove();
+  });
+
+  surfaceTest(
+    "mirrors video canvas after a seek via EFTimegroup",
+    async ({ timegroup, video, surface, expect }) => {
+      // Ensure media engine initialized
+      await video.mediaEngineTask.run();
+
+      // Seek to a known time through timegroup (triggers frame tasks)
+      timegroup.currentTimeMs = 3000;
+      await timegroup.seekTask.taskComplete;
+
+      // After scheduling, surface should have mirrored pixel dimensions
+      const videoCanvas = (video as any).canvasElement as
+        | HTMLCanvasElement
+        | undefined;
+      const surfaceCanvas =
+        (surface.shadowRoot?.querySelector("canvas") as HTMLCanvasElement) ??
+        undefined;
+
+      expect(videoCanvas).toBeTruthy();
+      expect(surfaceCanvas).toBeTruthy();
+      expect(videoCanvas!.width).toBeGreaterThan(0);
+      expect(videoCanvas!.height).toBeGreaterThan(0);
+
+      // Surface copies pixel dimensions
+      expect(surfaceCanvas!.width).toBe(videoCanvas!.width);
+      expect(surfaceCanvas!.height).toBe(videoCanvas!.height);
+    },
+  );
+
+  surfaceTest(
+    "supports multiple surfaces mirroring the same source",
+    async ({ expect }) => {
+      const container = document.createElement("div");
+      render(
+        html`
+        <ef-configuration api-host="http://localhost:63315">
+          <ef-preview>
+            <ef-timegroup mode="sequence" class="relative h-[360px] w-[640px] overflow-hidden bg-black">
+              <ef-video id="v" src="bars-n-tone.mp4" style="width: 100%; height: 100%;"></ef-video>
+              <ef-surface id="s1" target="v" style="position: absolute; inset: 0;"></ef-surface>
+              <ef-surface id="s2" target="v" style="position: absolute; inset: 0;"></ef-surface>
+            </ef-timegroup>
+          </ef-preview>
+        </ef-configuration>
+      `,
+        container,
+      );
+      document.body.appendChild(container);
+      const configuration = container.querySelector("ef-configuration") as any;
+      configuration.signingURL = "";
+      const timegroup = container.querySelector("ef-timegroup") as EFTimegroup;
+      const video = container.querySelector("ef-video") as EFVideo;
+      const s1 = container.querySelector("#s1") as unknown as EFSurface;
+      const s2 = container.querySelector("#s2") as unknown as EFSurface;
+      await timegroup.updateComplete;
+      await video.mediaEngineTask.run();
+
+      timegroup.currentTimeMs = 1000;
+      await timegroup.seekTask.taskComplete;
+
+      const vCanvas = (video as any).canvasElement as HTMLCanvasElement;
+      const c1 = s1.shadowRoot!.querySelector("canvas") as HTMLCanvasElement;
+      const c2 = s2.shadowRoot!.querySelector("canvas") as HTMLCanvasElement;
+
+      expect(vCanvas.width).toBeGreaterThan(0);
+      expect(c1.width).toBe(vCanvas.width);
+      expect(c2.width).toBe(vCanvas.width);
+      expect(c1.height).toBe(vCanvas.height);
+      expect(c2.height).toBe(vCanvas.height);
+
+      container.remove();
+    },
+  );
+
+  surfaceTest(
+    "handles missing video gracefully (no throw)",
+    async ({ expect }) => {
+      const el = document.createElement("ef-surface") as any;
+      document.body.appendChild(el);
+      await el.updateComplete;
+      await expect(el.frameTask.run()).resolves.toBeUndefined();
+      el.remove();
+    },
+  );
+});