@editframe/elements 0.17.6-beta.0 → 0.18.7-beta.0

package/src/elements/EFMedia/shared/BufferUtils.ts

@@ -0,0 +1,310 @@
+import type {
+  AudioRendition,
+  VideoRendition,
+} from "../../../transcoding/types";
+
+/**
+ * State interface for media buffering - generic for both audio and video
+ */
+export interface MediaBufferState {
+  currentSeekTimeMs: number;
+  activeRequests: Set<number>;
+  cachedSegments: Set<number>;
+  requestQueue: number[];
+}
+
+/**
+ * Configuration interface for media buffering - generic for both audio and video
+ */
+export interface MediaBufferConfig {
+  bufferDurationMs: number;
+  maxParallelFetches: number;
+  enableBuffering: boolean;
+  enableContinuousBuffering?: boolean;
+}
+
+/**
+ * Dependencies interface for media buffering - generic for both audio and video
+ */
+export interface MediaBufferDependencies<
+  T extends AudioRendition | VideoRendition,
+> {
+  computeSegmentId: (
+    timeMs: number,
+    rendition: T,
+  ) => Promise<number | undefined>;
+  fetchSegment: (segmentId: number, rendition: T) => Promise<ArrayBuffer>;
+  getRendition: () => Promise<T>;
+  logError: (message: string, error: any) => void;
+}
+
+/**
+ * Compute segment range for a time window
+ * Pure function - determines which segments are needed for a time range
+ */
+export const computeSegmentRange = <T extends AudioRendition | VideoRendition>(
+  startTimeMs: number,
+  endTimeMs: number,
+  rendition: T,
+  computeSegmentId: (timeMs: number, rendition: T) => number | undefined,
+): number[] => {
+  const segments: number[] = [];
+  const segmentDurationMs = (rendition as any).segmentDurationMs || 1000;
+
+  // Calculate segment indices that overlap with [startTimeMs, endTimeMs]
+  const startSegmentIndex = Math.floor(startTimeMs / segmentDurationMs);
+  const endSegmentIndex = Math.floor(endTimeMs / segmentDurationMs);
+
+  for (let i = startSegmentIndex; i <= endSegmentIndex; i++) {
+    const segmentId = computeSegmentId(i * segmentDurationMs, rendition);
+    if (segmentId !== undefined) {
+      segments.push(segmentId);
+    }
+  }
+
+  return segments.filter((id, index, arr) => arr.indexOf(id) === index); // Remove duplicates
+};
+
+/**
+ * Async version of computeSegmentRange for when computeSegmentId is async
+ */
+export const computeSegmentRangeAsync = async <
+  T extends AudioRendition | VideoRendition,
+>(
+  startTimeMs: number,
+  endTimeMs: number,
+  durationMs: number,
+  rendition: T,
+  computeSegmentId: (
+    timeMs: number,
+    rendition: T,
+  ) => Promise<number | undefined>,
+): Promise<number[]> => {
+  const segments: number[] = [];
+  const segmentDurationMs = (rendition as any).segmentDurationMs || 1000;
+
+  // Calculate segment indices that overlap with [startTimeMs, endTimeMs]
+  const startSegmentIndex = Math.floor(startTimeMs / segmentDurationMs);
+  const endSegmentIndex = Math.floor(
+    Math.min(endTimeMs, durationMs) / segmentDurationMs,
+  );
+
+  for (let i = startSegmentIndex; i <= endSegmentIndex; i++) {
+    const timeMs = i * segmentDurationMs;
+    if (timeMs < durationMs) {
+      const segmentId = await computeSegmentId(timeMs, rendition);
+      if (segmentId !== undefined) {
+        segments.push(segmentId);
+      }
+    }
+  }
+
+  return segments.filter((id, index, arr) => arr.indexOf(id) === index); // Remove duplicates
+};
+
+/**
+ * Compute buffer queue based on current state and desired segments
+ * Pure function - determines what segments should be fetched
+ */
+export const computeBufferQueue = (
+  desiredSegments: number[],
+  activeRequests: Set<number>,
+  cachedSegments: Set<number>,
+): number[] => {
+  return desiredSegments.filter(
+    (segmentId) =>
+      !activeRequests.has(segmentId) && !cachedSegments.has(segmentId),
+  );
+};
+
+/**
+ * Handle seek time change and recompute buffer queue
+ * Pure function - computes new queue when seek time changes
+ */
+export const handleSeekTimeChange = <T extends AudioRendition | VideoRendition>(
+  newSeekTimeMs: number,
+  bufferDurationMs: number,
+  rendition: T,
+  currentState: MediaBufferState,
+  computeSegmentId: (timeMs: number, rendition: T) => number | undefined,
+): { newQueue: number[]; overlappingRequests: number[] } => {
+  const endTimeMs = newSeekTimeMs + bufferDurationMs;
+  const desiredSegments = computeSegmentRange(
+    newSeekTimeMs,
+    endTimeMs,
+    rendition,
+    computeSegmentId,
+  );
+
+  // Find segments that are already being requested
+  const overlappingRequests = desiredSegments.filter((segmentId) =>
+    currentState.activeRequests.has(segmentId),
+  );
+
+  const newQueue = computeBufferQueue(
+    desiredSegments,
+    currentState.activeRequests,
+    currentState.cachedSegments,
+  );
+
+  return { newQueue, overlappingRequests };
+};
+
+/**
+ * Check if a specific segment is cached in the buffer
+ * Pure function for accessing buffer cache state
+ */
+export const getCachedSegment = (
+  segmentId: number,
+  bufferState: MediaBufferState | undefined,
+): boolean => {
+  return bufferState?.cachedSegments.has(segmentId) ?? false;
+};
+
+/**
+ * Get cached segments from a list of segment IDs
+ * Pure function that returns which segments are available in cache
+ */
+export const getCachedSegments = (
+  segmentIds: number[],
+  bufferState: MediaBufferState | undefined,
+): Set<number> => {
+  if (!bufferState) {
+    return new Set();
+  }
+  return new Set(segmentIds.filter((id) => bufferState.cachedSegments.has(id)));
+};
+
+/**
+ * Get missing segments from a list of segment IDs
+ * Pure function that returns which segments need to be fetched
+ */
+export const getMissingSegments = (
+  segmentIds: number[],
+  bufferState: MediaBufferState | undefined,
+): number[] => {
+  if (!bufferState) {
+    return segmentIds;
+  }
+  return segmentIds.filter((id) => !bufferState.cachedSegments.has(id));
+};
+
+/**
+ * Core media buffering logic with explicit dependencies
+ * Generic implementation that works for both audio and video
+ */
+export const manageMediaBuffer = async <
+  T extends AudioRendition | VideoRendition,
+>(
+  seekTimeMs: number,
+  config: MediaBufferConfig,
+  currentState: MediaBufferState,
+  durationMs: number,
+  signal: AbortSignal,
+  deps: MediaBufferDependencies<T>,
+): Promise<MediaBufferState> => {
+  if (!config.enableBuffering) {
+    return currentState;
+  }
+
+  const rendition = await deps.getRendition();
+  const endTimeMs = seekTimeMs + config.bufferDurationMs;
+
+  const desiredSegments = await computeSegmentRangeAsync(
+    seekTimeMs,
+    endTimeMs,
+    durationMs,
+    rendition,
+    deps.computeSegmentId,
+  );
+
+  const newQueue = computeBufferQueue(
+    desiredSegments,
+    currentState.activeRequests,
+    currentState.cachedSegments,
+  );
+
+  // Start fetching segments up to maxParallelFetches limit
+  const segmentsToFetch = newQueue.slice(0, config.maxParallelFetches);
+  const newActiveRequests = new Set(currentState.activeRequests);
+  const newCachedSegments = new Set(currentState.cachedSegments);
+
+  // Function to start next individual segment when a slot becomes available
+  const startNextSegment = (remainingQueue: number[]): void => {
+    if (remainingQueue.length === 0 || signal.aborted) return;
+
+    const availableSlots = config.maxParallelFetches - newActiveRequests.size;
+    if (availableSlots <= 0) return;
+
+    const nextSegmentId = remainingQueue[0];
+    if (nextSegmentId === undefined) return;
+
+    if (
+      newActiveRequests.has(nextSegmentId) ||
+      newCachedSegments.has(nextSegmentId)
+    ) {
+      // Skip this segment and try the next
+      startNextSegment(remainingQueue.slice(1));
+      return;
+    }
+
+    newActiveRequests.add(nextSegmentId);
+
+    deps
+      .fetchSegment(nextSegmentId, rendition)
+      .then(() => {
+        if (signal.aborted) return;
+        newActiveRequests.delete(nextSegmentId);
+        newCachedSegments.add(nextSegmentId);
+        startNextSegment(remainingQueue.slice(1));
+      })
+      .catch((error) => {
+        if (signal.aborted) return;
+        newActiveRequests.delete(nextSegmentId);
+        deps.logError(`Failed to fetch segment ${nextSegmentId}`, error);
+        startNextSegment(remainingQueue.slice(1));
+      });
+  };
+
+  // Start fetch promises for new segments
+  for (const segmentId of segmentsToFetch) {
+    if (signal.aborted) break;
+
+    newActiveRequests.add(segmentId);
+
+    // Start fetch (don't await - let it run in background)
+    deps
+      .fetchSegment(segmentId, rendition)
+      .then(() => {
+        if (signal.aborted) return;
+        // On success, move from active to cached
+        newActiveRequests.delete(segmentId);
+        newCachedSegments.add(segmentId);
+
+        // Continue buffering if there are more segments needed and continuous buffering is enabled
+        if (config.enableContinuousBuffering ?? true) {
+          const remainingQueue = newQueue.slice(segmentsToFetch.length);
+          startNextSegment(remainingQueue);
+        }
+      })
+      .catch((error) => {
+        if (signal.aborted) return;
+        // On error, remove from active and continue
+        newActiveRequests.delete(segmentId);
+        deps.logError(`Failed to fetch segment ${segmentId}`, error);
+
+        // Continue buffering even after error if continuous buffering is enabled
+        if (config.enableContinuousBuffering ?? true) {
+          const remainingQueue = newQueue.slice(segmentsToFetch.length);
+          startNextSegment(remainingQueue);
+        }
+      });
+  }
+
+  return {
+    currentSeekTimeMs: seekTimeMs,
+    activeRequests: newActiveRequests,
+    cachedSegments: newCachedSegments,
+    requestQueue: newQueue.slice(segmentsToFetch.length), // Remaining queue
+  };
+};

package/src/elements/EFMedia/shared/MediaTaskUtils.ts

@@ -0,0 +1,44 @@
+import type { Task } from "@lit/task";
+import type {
+  AudioRendition,
+  MediaEngine,
+  VideoRendition,
+} from "../../../transcoding/types";
+import type { BufferedSeekingInput } from "../BufferedSeekingInput";
+
+/**
+ * Generic rendition type that can be either audio or video
+ */
+export type MediaRendition = AudioRendition | VideoRendition;
+
+/**
+ * Generic task type for init segment fetch
+ */
+export type InitSegmentFetchTask = Task<
+  readonly [MediaEngine | undefined],
+  ArrayBuffer
+>;
+
+/**
+ * Generic task type for segment ID calculation
+ */
+export type SegmentIdTask = Task<
+  readonly [MediaEngine | undefined, number],
+  number | undefined
+>;
+
+/**
+ * Generic task type for segment fetch
+ */
+export type SegmentFetchTask = Task<
+  readonly [MediaEngine | undefined, number | undefined],
+  ArrayBuffer
+>;
+
+/**
+ * Generic task type for input creation
+ */
+export type InputTask = Task<
+  readonly [ArrayBuffer, ArrayBuffer],
+  BufferedSeekingInput
+>;

package/src/elements/EFMedia/shared/PrecisionUtils.ts

@@ -0,0 +1,46 @@
+/**
+ * Centralized precision utilities for consistent timing calculations across the media pipeline.
+ *
+ * The key insight is that floating-point precision errors can cause inconsistencies between:
+ * 1. Segment selection logic (in AssetMediaEngine.computeSegmentId)
+ * 2. Sample finding logic (in SampleBuffer.find)
+ * 3. Timeline mapping (in BufferedSeekingInput.seek)
+ *
+ * All timing calculations must use the same rounding strategy to ensure consistency.
+ */
+
+/**
+ * Round time to millisecond precision to handle floating-point precision issues.
+ * Uses Math.round for consistent behavior across the entire pipeline.
+ *
+ * This function should be used for ALL time-related calculations that need to be
+ * compared between different parts of the system.
+ */
+export const roundToMilliseconds = (timeMs: number): number => {
+  // Round to 3 decimal places (microsecond precision)
+  return Math.round(timeMs * 1000) / 1000;
+};
+
+/**
+ * Convert media time (in seconds) to scaled time units using consistent rounding.
+ * This is used in segment selection to convert from milliseconds to timescale units.
+ */
+export const convertToScaledTime = (
+  timeMs: number,
+  timescale: number,
+): number => {
+  const scaledTime = (timeMs / 1000) * timescale;
+  return Math.round(scaledTime);
+};
+
+/**
+ * Convert scaled time units back to media time (in milliseconds) using consistent rounding.
+ * This is the inverse of convertToScaledTime.
+ */
+export const convertFromScaledTime = (
+  scaledTime: number,
+  timescale: number,
+): number => {
+  const timeMs = (scaledTime / timescale) * 1000;
+  return roundToMilliseconds(timeMs);
+};

package/src/elements/EFMedia/shared/RenditionHelpers.browsertest.ts

@@ -0,0 +1,247 @@
+import { test as baseTest, describe, vi } from "vitest";
+import type {
+  AudioRendition,
+  MediaEngine,
+  VideoRendition,
+} from "../../../transcoding/types";
+import {
+  calculateSegmentRange,
+  computeSegmentId,
+  getAudioRendition,
+  getVideoRendition,
+} from "./RenditionHelpers";
+
+const test = baseTest.extend<{
+  mockMediaEngine: MediaEngine;
+  mockVideoRendition: VideoRendition;
+  mockAudioRendition: AudioRendition;
+  mockMediaEngineWithoutAudio: MediaEngine;
+  mockMediaEngineWithoutVideo: MediaEngine;
+}>({
+  mockMediaEngine: async ({}, use) => {
+    const mockVideoRendition = {
+      trackId: 1,
+      src: "video-track.mp4",
+      segmentDurationMs: 1000,
+    } as VideoRendition;
+
+    const mockAudioRendition = {
+      trackId: 2,
+      src: "audio-track.mp4",
+      segmentDurationMs: 1000,
+    } as AudioRendition;
+
+    const mockMediaEngine = {
+      durationMs: 10000,
+      src: "https://example.com/media.mp4",
+      videoRendition: mockVideoRendition,
+      audioRendition: mockAudioRendition,
+      fetchMediaSegment: vi.fn(),
+    } as unknown as MediaEngine;
+    await use(mockMediaEngine);
+  },
+
+  mockVideoRendition: async ({}, use) => {
+    const mockVideoRendition = {
+      trackId: 1,
+      src: "video-track.mp4",
+      segmentDurationMs: 1000,
+    } as VideoRendition;
+    await use(mockVideoRendition);
+  },
+
+  mockAudioRendition: async ({}, use) => {
+    const mockAudioRendition = {
+      trackId: 2,
+      src: "audio-track.mp4",
+      segmentDurationMs: 1000,
+    } as AudioRendition;
+    await use(mockAudioRendition);
+  },
+
+  mockMediaEngineWithoutAudio: async ({}, use) => {
+    const mockMediaEngine = {
+      durationMs: 10000,
+      src: "https://example.com/media.mp4",
+      videoRendition: {
+        trackId: 1,
+        src: "video-track.mp4",
+        segmentDurationMs: 1000,
+      } as VideoRendition,
+      audioRendition: null,
+      fetchMediaSegment: vi.fn(),
+    } as unknown as MediaEngine;
+    await use(mockMediaEngine);
+  },
+
+  mockMediaEngineWithoutVideo: async ({}, use) => {
+    const mockMediaEngine = {
+      durationMs: 10000,
+      src: "https://example.com/media.mp4",
+      videoRendition: null,
+      audioRendition: {
+        trackId: 2,
+        src: "audio-track.mp4",
+        segmentDurationMs: 1000,
+      } as AudioRendition,
+      fetchMediaSegment: vi.fn(),
+    } as unknown as MediaEngine;
+    await use(mockMediaEngine);
+  },
+});
+
+describe("RenditionHelpers", () => {
+  describe("getAudioRendition", () => {
+    test("returns audio rendition when available", ({
+      mockMediaEngine,
+      expect,
+    }) => {
+      const result = getAudioRendition(mockMediaEngine);
+      expect(result).toBe(mockMediaEngine.audioRendition);
+      expect(result.trackId).toBe(2);
+      expect(result.src).toBe("audio-track.mp4");
+    });
+
+    test("throws error when audio rendition is not available", ({
+      mockMediaEngineWithoutAudio,
+      expect,
+    }) => {
+      expect(() => getAudioRendition(mockMediaEngineWithoutAudio)).toThrow(
+        "Audio rendition is not available",
+      );
+    });
+  });
+
+  describe("getVideoRendition", () => {
+    test("returns video rendition when available", ({
+      mockMediaEngine,
+      expect,
+    }) => {
+      const result = getVideoRendition(mockMediaEngine);
+      expect(result).toBe(mockMediaEngine.videoRendition);
+      expect(result.trackId).toBe(1);
+      expect(result.src).toBe("video-track.mp4");
+    });
+
+    test("throws error when video rendition is not available", ({
+      mockMediaEngineWithoutVideo,
+      expect,
+    }) => {
+      expect(() => getVideoRendition(mockMediaEngineWithoutVideo)).toThrow(
+        "Video rendition is not available",
+      );
+    });
+  });
+
+  describe("computeSegmentId", () => {
+    test("calculates segment ID correctly for audio rendition", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      // Test various time points
+      expect(computeSegmentId(0, mockAudioRendition)).toBe(1); // First segment
+      expect(computeSegmentId(500, mockAudioRendition)).toBe(1); // Still first segment
+      expect(computeSegmentId(999, mockAudioRendition)).toBe(1); // Still first segment
+      expect(computeSegmentId(1000, mockAudioRendition)).toBe(2); // Second segment
+      expect(computeSegmentId(1500, mockAudioRendition)).toBe(2); // Still second segment
+      expect(computeSegmentId(2000, mockAudioRendition)).toBe(3); // Third segment
+    });
+
+    test("calculates segment ID correctly for video rendition", ({
+      mockVideoRendition,
+      expect,
+    }) => {
+      // Test various time points
+      expect(computeSegmentId(0, mockVideoRendition)).toBe(1); // First segment
+      expect(computeSegmentId(999, mockVideoRendition)).toBe(1); // Still first segment
+      expect(computeSegmentId(1000, mockVideoRendition)).toBe(2); // Second segment
+      expect(computeSegmentId(2500, mockVideoRendition)).toBe(3); // Third segment
+    });
+
+    test("returns undefined when segmentDurationMs is not available", ({
+      expect,
+    }) => {
+      const renditionWithoutDuration = {
+        trackId: 1,
+        src: "test.mp4",
+        segmentDurationMs: undefined,
+      } as AudioRendition;
+
+      expect(computeSegmentId(1000, renditionWithoutDuration)).toBeUndefined();
+    });
+
+    test("handles edge case of negative time", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      expect(computeSegmentId(-100, mockAudioRendition)).toBe(1); // Should clamp to segment 1
+    });
+  });
+
+  describe("calculateSegmentRange", () => {
+    test("calculates segment range for single segment", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      const result = calculateSegmentRange(100, 800, mockAudioRendition);
+      expect(result).toEqual([1]);
+    });
+
+    test("calculates segment range spanning multiple segments", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      const result = calculateSegmentRange(500, 2500, mockAudioRendition);
+      expect(result).toEqual([1, 2, 3]);
+    });
+
+    test("calculates segment range for exact segment boundaries", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      const result = calculateSegmentRange(1000, 2000, mockAudioRendition);
+      expect(result).toEqual([2, 3]);
+    });
+
+    test("handles single time point (start equals end)", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      const result = calculateSegmentRange(1500, 1500, mockAudioRendition);
+      expect(result).toEqual([2]);
+    });
+
+    test("returns empty array when segmentDurationMs is not available", ({
+      expect,
+    }) => {
+      const renditionWithoutDuration = {
+        trackId: 1,
+        src: "test.mp4",
+        segmentDurationMs: undefined,
+      } as AudioRendition;
+
+      const result = calculateSegmentRange(
+        1000,
+        2000,
+        renditionWithoutDuration,
+      );
+      expect(result).toEqual([]);
+    });
+
+    test("works with video renditions too", ({
+      mockVideoRendition,
+      expect,
+    }) => {
+      const result = calculateSegmentRange(1500, 3500, mockVideoRendition);
+      expect(result).toEqual([2, 3, 4]);
+    });
+
+    test("handles edge case where start time is negative", ({
+      mockAudioRendition,
+      expect,
+    }) => {
+      const result = calculateSegmentRange(-500, 1500, mockAudioRendition);
+      expect(result).toEqual([1, 2]);
+    });
+  });
+});