@editframe/elements 0.17.6-beta.0 → 0.18.3-beta.0

package/src/elements/EFMedia/BaseMediaEngine.ts

@@ -0,0 +1,170 @@
+import { RequestDeduplicator } from "../../transcoding/cache/RequestDeduplicator.js";
+import type {
+  AudioRendition,
+  SegmentTimeRange,
+  VideoRendition,
+} from "../../transcoding/types";
+import { SizeAwareLRUCache } from "../../utils/LRUCache.js";
+import type { EFMedia } from "../EFMedia.js";
+
+// 100MB cache limit
+const mediaCache = new SizeAwareLRUCache<string>(100 * 1024 * 1024);
+
+export abstract class BaseMediaEngine {
+  // Request deduplicator for in-flight segment fetches
+  private requestDeduplicator = new RequestDeduplicator();
+
+  abstract get videoRendition(): VideoRendition | undefined;
+  abstract get audioRendition(): AudioRendition | undefined;
+  abstract get host(): EFMedia;
+
+  getVideoRendition(): VideoRendition {
+    if (!this.videoRendition) {
+      throw new Error("No video rendition available");
+    }
+    return this.videoRendition;
+  }
+
+  getAudioRendition(): AudioRendition {
+    if (!this.audioRendition) {
+      throw new Error("No audio rendition available");
+    }
+    return this.audioRendition;
+  }
+
+  /**
+   * Generate cache key for segment requests
+   */
+  private getSegmentCacheKey(
+    segmentId: number,
+    rendition: { src: string; trackId: number | undefined; id?: string },
+  ): string {
+    return `${rendition.src}-${rendition.id}-${segmentId}-${rendition.trackId}`;
+  }
+
+  /**
+   * Abstract method for actual segment fetching - implemented by subclasses
+   */
+  abstract fetchMediaSegmentImpl(
+    segmentId: number,
+    rendition: { trackId: number | undefined; src: string },
+  ): Promise<ArrayBuffer>;
+
+  /**
+   * Fetch media segment with built-in deduplication
+   * Eliminates the need for separate coordinators - cleaner architecture
+   */
+  async fetchMediaSegment(
+    segmentId: number,
+    rendition: { trackId: number | undefined; src: string },
+    _signal?: AbortSignal,
+  ): Promise<ArrayBuffer> {
+    const cacheKey = this.getSegmentCacheKey(segmentId, rendition);
+
+    return this.requestDeduplicator.executeRequest(cacheKey, async () => {
+      return this.fetchMediaSegmentImpl(segmentId, rendition);
+    });
+  }
+
+  /**
+   * Check if a segment is currently being fetched
+   */
+  isSegmentBeingFetched(
+    segmentId: number,
+    rendition: { src: string; trackId: number | undefined },
+  ): boolean {
+    const cacheKey = this.getSegmentCacheKey(segmentId, rendition);
+    return this.requestDeduplicator.isPending(cacheKey);
+  }
+
+  /**
+   * Get count of active segment requests (for debugging/monitoring)
+   */
+  getActiveSegmentRequestCount(): number {
+    return this.requestDeduplicator.getPendingCount();
+  }
+
+  /**
+   * Cancel all active segment requests (for cleanup)
+   */
+  cancelAllSegmentRequests(): void {
+    this.requestDeduplicator.clear();
+  }
+
+  async fetchMediaCache(mediaUrl: string) {
+    const cached = mediaCache.get(mediaUrl);
+    if (cached) {
+      return cached;
+    }
+    const promise = this.host
+      .fetch(mediaUrl)
+      .then((response) => response.arrayBuffer());
+    mediaCache.set(mediaUrl, promise);
+    return promise;
+  }
+
+  /**
+   * Enhanced caching method that supports custom headers (e.g., Range requests)
+   * Cache key includes both URL and headers for proper cache isolation
+   */
+  async fetchMediaCacheWithHeaders(
+    mediaUrl: string,
+    headers?: Record<string, string>,
+    signal?: AbortSignal,
+  ) {
+    // Create a cache key that includes both URL and headers
+    const cacheKey = headers
+      ? `${mediaUrl}:${JSON.stringify(headers)}`
+      : mediaUrl;
+
+    const cached = mediaCache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+
+    const promise = this.host
+      .fetch(mediaUrl, { headers, signal })
+      .then((response) => response.arrayBuffer());
+    mediaCache.set(cacheKey, promise);
+    return promise;
+  }
+
+  /**
+   * Calculate audio segments needed for a time range
+   * Each media engine implements this based on their segment structure
+   */
+  calculateAudioSegmentRange(
+    fromMs: number,
+    toMs: number,
+    rendition: AudioRendition,
+    durationMs: number,
+  ): SegmentTimeRange[] {
+    // Default implementation for uniform segments (used by JitMediaEngine)
+    if (fromMs >= toMs) {
+      return [];
+    }
+
+    const segmentDurationMs = rendition.segmentDurationMs || 1000;
+    const segments: SegmentTimeRange[] = [];
+
+    const startSegmentIndex = Math.floor(fromMs / segmentDurationMs);
+    const endSegmentIndex = Math.floor(toMs / segmentDurationMs);
+
+    for (let i = startSegmentIndex; i <= endSegmentIndex; i++) {
+      const segmentId = i + 1; // Convert to 1-based
+      const segmentStartMs = i * segmentDurationMs;
+      const segmentEndMs = Math.min((i + 1) * segmentDurationMs, durationMs);
+
+      // Only include segments that overlap with requested time range
+      if (segmentStartMs < toMs && segmentEndMs > fromMs) {
+        segments.push({
+          segmentId,
+          startMs: segmentStartMs,
+          endMs: segmentEndMs,
+        });
+      }
+    }
+
+    return segments;
+  }
+}

package/src/elements/EFMedia/BufferedSeekingInput.browsertest.ts

@@ -0,0 +1,400 @@
+import { test as baseTest, describe } from "vitest";
+import { BufferedSeekingInput, NoSample } from "./BufferedSeekingInput";
+
+const test = baseTest.extend<{
+  fiveSampleBuffer: BufferedSeekingInput;
+  inputAtStart: BufferedSeekingInput;
+  inputAtMiddle: BufferedSeekingInput;
+  segment2: BufferedSeekingInput;
+}>({
+  fiveSampleBuffer: async ({}, use) => {
+    const response = await fetch("/jit-segments/segment-0ms-2s-low.mp4");
+    const arrayBuffer = await response.arrayBuffer();
+    const input = new BufferedSeekingInput(arrayBuffer, {
+      videoBufferSize: 5,
+      audioBufferSize: 5,
+    });
+    await use(input);
+  },
+  inputAtStart: async ({}, use) => {
+    const response = await fetch("/jit-segments/segment-0ms-2s-low.mp4");
+    const arrayBuffer = await response.arrayBuffer();
+    const input = new BufferedSeekingInput(arrayBuffer);
+    await use(input);
+  },
+  inputAtMiddle: async ({}, use) => {
+    const response = await fetch("/jit-segments/segment-6000ms-1s-low.mp4");
+    const arrayBuffer = await response.arrayBuffer();
+    const input = new BufferedSeekingInput(arrayBuffer);
+    await use(input);
+  },
+  segment2: async ({}, use) => {
+    const response = await fetch("/jit-segments/segment-2.mp4");
+    const arrayBuffer = await response.arrayBuffer();
+    const input = new BufferedSeekingInput(arrayBuffer);
+    await use(input);
+  },
+});
+
+describe("BufferedSeekingInput", () => {
+  describe("computeDuration", () => {
+    test("computes duration", async ({
+      expect,
+      inputAtStart,
+      inputAtMiddle,
+    }) => {
+      await expect(inputAtStart.computeDuration()).resolves.toBe(2);
+      await expect(inputAtMiddle.computeDuration()).resolves.toBeCloseTo(0.96);
+    });
+  });
+
+  describe("basic seeking", () => {
+    test("seeks to frame at 0 seconds", async ({ expect, inputAtStart }) => {
+      const sample = await inputAtStart.seek(1, 0);
+      expect(sample.timestamp).toBe(0);
+    });
+
+    test("seeks to frame at 0.02 seconds", async ({ expect, inputAtStart }) => {
+      const sample = await inputAtStart.seek(1, 20);
+      expect(sample.timestamp).toBe(0);
+    });
+
+    test("seeks to frame at 0.04 seconds", async ({ expect, inputAtStart }) => {
+      const sample = await inputAtStart.seek(1, 40);
+      expect(sample.timestamp).toBe(0.04);
+    });
+  });
+
+  describe("deterministic seeking behavior", () => {
+    test("seeks to exact sample timestamps", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      expect((await inputAtStart.seek(1, 0)).timestamp).toBe(0);
+      expect((await inputAtStart.seek(1, 40)).timestamp).toBe(0.04);
+      expect((await inputAtStart.seek(1, 80)).timestamp).toBe(0.08);
+      expect((await inputAtStart.seek(1, 120)).timestamp).toBe(0.12);
+      expect((await inputAtStart.seek(1, 160)).timestamp).toBe(0.16);
+    });
+
+    test("seeks between samples returns previous sample", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      expect((await inputAtStart.seek(1, 30)).timestamp).toBe(0);
+      expect((await inputAtStart.seek(1, 60)).timestamp).toBe(0.04);
+      expect((await inputAtStart.seek(1, 100)).timestamp).toBe(0.08);
+      expect((await inputAtStart.seek(1, 140)).timestamp).toBe(0.12);
+    });
+
+    test("seeks before first sample", async ({ expect, inputAtStart }) => {
+      inputAtStart.clearBuffer(1);
+      expect((await inputAtStart.seek(1, 0)).timestamp).toBe(0);
+    });
+
+    test("seeks to later samples in media", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      const result200 = await inputAtStart.seek(1, 200);
+      const result1000 = await inputAtStart.seek(1, 1000);
+
+      expect(result200.timestamp! * 1000).toBeLessThanOrEqual(200);
+      expect(result1000.timestamp! * 1000).toBeLessThanOrEqual(1000);
+      expect(result200.timestamp).toBeGreaterThanOrEqual(0);
+      expect(result1000.timestamp).toBeGreaterThanOrEqual(result200.timestamp!);
+    });
+
+    test("never returns future sample", async ({ expect, inputAtStart }) => {
+      const testCases = [
+        { seekTimeMs: 0, expectedTimestamp: 0 },
+        { seekTimeMs: 10, expectedTimestamp: 0 },
+        { seekTimeMs: 30, expectedTimestamp: 0 },
+        { seekTimeMs: 40, expectedTimestamp: 0.04 },
+        { seekTimeMs: 50, expectedTimestamp: 0.04 },
+        { seekTimeMs: 70, expectedTimestamp: 0.04 },
+        { seekTimeMs: 80, expectedTimestamp: 0.08 },
+        { seekTimeMs: 90, expectedTimestamp: 0.08 },
+      ];
+
+      for (const { seekTimeMs, expectedTimestamp } of testCases) {
+        const result = await inputAtStart.seek(1, seekTimeMs);
+        expect(result.timestamp).toBe(expectedTimestamp);
+
+        const resultTimeMs = result.timestamp! * 1000;
+        expect(resultTimeMs).toBeLessThanOrEqual(seekTimeMs);
+      }
+    });
+  });
+
+  describe("buffer state management", () => {
+    test("starts with empty buffer", async ({ expect, inputAtStart }) => {
+      expect(inputAtStart.getBufferSize(1)).toBe(0);
+      expect(inputAtStart.getBufferTimestamps(1)).toEqual([]);
+      expect(inputAtStart.getBufferContents(1)).toEqual([]);
+    });
+
+    test("maintains separate buffers per track", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      await inputAtStart.seek(1, 0);
+      const track1BufferSize = inputAtStart.getBufferSize(1);
+      expect(track1BufferSize).toBeGreaterThan(0);
+
+      expect(inputAtStart.getBufferSize(2)).toBe(0);
+
+      await inputAtStart.seek(2, 0);
+      expect(inputAtStart.getBufferSize(2)).toBeGreaterThan(0);
+      expect(inputAtStart.getBufferSize(1)).toBe(track1BufferSize);
+    });
+
+    test("buffer accumulates samples in order", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      inputAtStart.clearBuffer(1);
+
+      await inputAtStart.seek(1, 0);
+      await inputAtStart.seek(1, 40);
+      await inputAtStart.seek(1, 80);
+
+      const timestamps = inputAtStart.getBufferTimestamps(1);
+      expect(timestamps).toContain(0);
+      expect(timestamps).toContain(0.04);
+      expect(timestamps).toContain(0.08);
+    });
+
+    test("buffer extends one sample ahead", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+    });
+
+    test("buffer resets when seeking back before the buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      await fiveSampleBuffer.seek(1, 0);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([0]);
+    });
+
+    test("buffer is maintained when seeking forwards within the buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 900);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+    });
+
+    test("buffer is maintained when seeking backwards within the buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 900);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+    });
+
+    test("buffer is maintained when seeking backwards to start of buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 800);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+    });
+
+    test("buffer is reset when seeking backwards to arbitrary time before buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 720);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.56, 0.6, 0.64, 0.68, 0.72,
+      ]);
+    });
+
+    test("buffer is maintained when seeking forwards to end of buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 900);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+    });
+
+    test("buffer is maintained when seeking forwards past the buffer", async ({
+      expect,
+      fiveSampleBuffer,
+    }) => {
+      await fiveSampleBuffer.seek(1, 960);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.8, 0.84, 0.88, 0.92, 0.96,
+      ]);
+      await fiveSampleBuffer.seek(1, 1000);
+      expect(fiveSampleBuffer.getBufferTimestamps(1)).toEqual([
+        0.84, 0.88, 0.92, 0.96, 1,
+      ]);
+    });
+  });
+
+  describe("seeing to time not in buffer (time is before buffer)", () => {
+    test("throws error", async ({ expect, segment2 }) => {
+      await expect(segment2.seek(1, 0)).rejects.toThrow(NoSample);
+    });
+  });
+
+  describe("seeking forward at 1ms intervals", () => {
+    test("returns all samples in the media", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      const timestamps = new Set<number>();
+      for (let i = 0; i < 1999; i++) {
+        const sample = await inputAtStart.seek(1, i);
+        timestamps.add(sample.timestamp!);
+      }
+      expect(Array.from(timestamps)).toEqual([
+        0, 0.04, 0.08, 0.12, 0.16, 0.2, 0.24, 0.28, 0.32, 0.36, 0.4, 0.44, 0.48,
+        0.52, 0.56, 0.6, 0.64, 0.68, 0.72, 0.76, 0.8, 0.84, 0.88, 0.92, 0.96, 1,
+        1.04, 1.08, 1.12, 1.16, 1.2, 1.24, 1.28, 1.32, 1.36, 1.4, 1.44, 1.48,
+        1.52, 1.56, 1.6, 1.64, 1.68, 1.72, 1.76, 1.8, 1.84, 1.88, 1.92, 1.96,
+      ]);
+    });
+  });
+
+  describe("error handling", () => {
+    test("throws error for non-existent track", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      await expect(inputAtStart.seek(999, 0)).rejects.toThrow(
+        "Track 999 not found",
+      );
+    });
+  });
+
+  describe("concurrency handling", () => {
+    test("handles concurrent seeking operations safely", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      const seek1 = inputAtStart.seek(1, 0);
+      const seek2 = inputAtStart.seek(1, 40);
+      const seek3 = inputAtStart.seek(1, 80);
+      const seek4 = inputAtStart.seek(1, 120);
+
+      const [sample1, sample2, sample3, sample4] = await Promise.all([
+        seek1,
+        seek2,
+        seek3,
+        seek4,
+      ]);
+
+      expect(sample1.timestamp).toBe(0);
+      expect(sample2.timestamp).toBe(0.04);
+      expect(sample3.timestamp).toBe(0.08);
+      expect(sample4.timestamp).toBe(0.12);
+
+      const bufferTimestamps = inputAtStart.getBufferTimestamps(1);
+      expect(bufferTimestamps.length).toBeGreaterThan(0);
+      for (let i = 1; i < bufferTimestamps.length; i++) {
+        expect(bufferTimestamps[i]).toBeGreaterThanOrEqual(
+          bufferTimestamps[i - 1]!,
+        );
+      }
+    });
+
+    test("handles concurrent seeks with backward jumps", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      await inputAtStart.seek(1, 200);
+
+      const seek1 = inputAtStart.seek(1, 40);
+      const seek2 = inputAtStart.seek(1, 160);
+      const seek3 = inputAtStart.seek(1, 0);
+
+      const [sample1, sample2, sample3] = await Promise.all([
+        seek1,
+        seek2,
+        seek3,
+      ]);
+
+      expect(sample1.timestamp).toBe(0.04);
+      expect(sample2.timestamp).toBe(0.16);
+      expect(sample3.timestamp).toBe(0);
+      expect(inputAtStart.getBufferSize(1)).toBeGreaterThan(0);
+    });
+
+    test("handles concurrent seeks to same time", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      const seeks = Array(5)
+        .fill(null)
+        .map(() => inputAtStart.seek(1, 80));
+      const results = await Promise.all(seeks);
+
+      for (const result of results) {
+        expect(result.timestamp).toBe(0.08);
+      }
+      expect(inputAtStart.getBufferSize(1)).toBeGreaterThan(0);
+    });
+
+    test("handles mixed concurrent operations across different tracks", async ({
+      expect,
+      inputAtStart,
+    }) => {
+      const track1Seek1 = inputAtStart.seek(1, 40);
+      const track1Seek2 = inputAtStart.seek(1, 80);
+
+      const [result1, result2] = await Promise.all([track1Seek1, track1Seek2]);
+
+      expect(result1.timestamp).toBe(0.04);
+      expect(result2.timestamp).toBe(0.08);
+
+      const track1Buffer = inputAtStart.getBufferTimestamps(1);
+      expect(track1Buffer.length).toBeGreaterThan(0);
+    });
+  });
+});