@editframe/elements 0.17.6-beta.0 → 0.18.7-beta.0

package/src/elements/EFMedia/BufferedSeekingInput.ts

@@ -0,0 +1,324 @@
+import {
+  AudioSampleSink,
+  BufferSource,
+  Input,
+  MP4,
+  VideoSampleSink,
+} from "mediabunny";
+import { type MediaSample, SampleBuffer } from "../SampleBuffer";
+import { roundToMilliseconds } from "./shared/PrecisionUtils";
+
+interface BufferedSeekingInputOptions {
+  videoBufferSize?: number;
+  audioBufferSize?: number;
+  /**
+   * Timeline offset in milliseconds to map user timeline to media timeline.
+   * Applied during seeking to handle media that doesn't start at 0ms.
+   */
+  startTimeOffsetMs?: number;
+}
+
+const defaultOptions: BufferedSeekingInputOptions = {
+  videoBufferSize: 30,
+  audioBufferSize: 100,
+  startTimeOffsetMs: 0,
+};
+
+export class NoSample extends RangeError {}
+
+export class BufferedSeekingInput {
+  private input: Input;
+  private trackIterators: Map<number, AsyncIterator<MediaSample>> = new Map();
+  private trackBuffers: Map<number, SampleBuffer> = new Map();
+  private options: BufferedSeekingInputOptions;
+  // Separate locks for different operation types to prevent unnecessary blocking
+  private trackIteratorCreationPromises: Map<number, Promise<any>> = new Map();
+  private trackSeekPromises: Map<number, Promise<any>> = new Map();
+
+  /**
+   * Timeline offset in milliseconds to map user timeline to media timeline.
+   * Applied during seeking to handle media that doesn't start at 0ms.
+   */
+  private readonly startTimeOffsetMs: number;
+
+  constructor(arrayBuffer: ArrayBuffer, options?: BufferedSeekingInputOptions) {
+    const bufferSource = new BufferSource(arrayBuffer);
+    const input = new Input({
+      source: bufferSource,
+      formats: [MP4],
+    });
+    this.input = input;
+    this.options = { ...defaultOptions, ...options };
+    this.startTimeOffsetMs = this.options.startTimeOffsetMs ?? 0;
+  }
+
+  // Buffer inspection API for testing
+  getBufferSize(trackId: number): number {
+    const buffer = this.trackBuffers.get(trackId);
+    return buffer ? buffer.length : 0;
+  }
+
+  getBufferContents(trackId: number): readonly MediaSample[] {
+    const buffer = this.trackBuffers.get(trackId);
+    return buffer ? Object.freeze([...buffer.getContents()]) : [];
+  }
+
+  getBufferTimestamps(trackId: number): number[] {
+    const contents = this.getBufferContents(trackId);
+    return contents.map((sample) => sample.timestamp || 0);
+  }
+
+  clearBuffer(trackId: number): void {
+    const buffer = this.trackBuffers.get(trackId);
+    if (buffer) {
+      buffer.clear();
+    }
+  }
+
+  computeDuration() {
+    return this.input.computeDuration();
+  }
+
+  async getTrack(trackId: number) {
+    const tracks = await this.input.getTracks();
+    const track = tracks.find((track) => track.id === trackId);
+    if (!track) {
+      throw new Error(`Track ${trackId} not found`);
+    }
+    return track;
+  }
+
+  async getAudioTrack(trackId: number) {
+    const tracks = await this.input.getAudioTracks();
+    const track = tracks.find(
+      (track) => track.id === trackId && track.type === "audio",
+    );
+    if (!track) {
+      throw new Error(`Track ${trackId} not found`);
+    }
+    return track;
+  }
+
+  async getVideoTrack(trackId: number) {
+    const tracks = await this.input.getVideoTracks();
+    const track = tracks.find(
+      (track) => track.id === trackId && track.type === "video",
+    );
+    if (!track) {
+      throw new Error(`Track ${trackId} not found`);
+    }
+    return track;
+  }
+
+  async getFirstVideoTrack() {
+    const tracks = await this.input.getVideoTracks();
+    return tracks[0];
+  }
+
+  async getFirstAudioTrack() {
+    const tracks = await this.input.getAudioTracks();
+    return tracks[0];
+  }
+
+  async getTrackIterator(trackId: number) {
+    if (this.trackIterators.has(trackId)) {
+      // biome-ignore lint/style/noNonNullAssertion: we know the map has the key
+      return this.trackIterators.get(trackId)!;
+    }
+
+    // Serialize iterator creation per track (but don't block seeks)
+    const existingIteratorCreation =
+      this.trackIteratorCreationPromises.get(trackId);
+    if (existingIteratorCreation) {
+      await existingIteratorCreation;
+      // Check again after waiting - another operation might have created it
+      if (this.trackIterators.has(trackId)) {
+        // biome-ignore lint/style/noNonNullAssertion: we know the map has the key
+        return this.trackIterators.get(trackId)!;
+      }
+    }
+
+    const creationPromise = this.createIteratorSafe(trackId);
+    this.trackIteratorCreationPromises.set(trackId, creationPromise);
+
+    try {
+      const iterator = await creationPromise;
+      return iterator;
+    } finally {
+      this.trackIteratorCreationPromises.delete(trackId);
+    }
+  }
+
+  private async createIteratorSafe(trackId: number) {
+    const track = await this.getTrack(trackId);
+    if (track.type === "audio") {
+      const track = await this.getAudioTrack(trackId);
+      const sampleSink = new AudioSampleSink(track);
+      const iterator = sampleSink.samples();
+      this.trackIterators.set(trackId, iterator);
+      return iterator;
+    }
+    {
+      const track = await this.getVideoTrack(trackId);
+      const sampleSink = new VideoSampleSink(track);
+      const iterator = sampleSink.samples();
+      this.trackIterators.set(trackId, iterator);
+      return iterator;
+    }
+  }
+
+  async createTrackBuffer(trackId: number) {
+    const track = await this.getTrack(trackId);
+    if (track.type === "audio") {
+      const bufferSize = this.options.audioBufferSize;
+      this.trackBuffers.set(trackId, new SampleBuffer(bufferSize));
+    } else {
+      const bufferSize = this.options.videoBufferSize;
+      this.trackBuffers.set(trackId, new SampleBuffer(bufferSize));
+    }
+  }
+
+  async seek(trackId: number, timeMs: number) {
+    // Apply timeline offset to map user timeline to media timeline
+    const mediaTimeMs = timeMs + this.startTimeOffsetMs;
+
+    // Round using consistent precision handling
+    const roundedMediaTimeMs = roundToMilliseconds(mediaTimeMs);
+
+    // Serialize seek operations per track (but don't block iterator creation)
+    const existingSeek = this.trackSeekPromises.get(trackId);
+    if (existingSeek) {
+      await existingSeek;
+    }
+
+    const seekPromise = this.seekSafe(trackId, roundedMediaTimeMs);
+    this.trackSeekPromises.set(trackId, seekPromise);
+
+    try {
+      return await seekPromise;
+    } finally {
+      this.trackSeekPromises.delete(trackId);
+    }
+  }
+
+  private async resetIterator(trackId: number) {
+    const trackBuffer = this.trackBuffers.get(trackId);
+    trackBuffer?.clear();
+    // Clean up iterator safely - wait for any ongoing iterator creation
+    const ongoingIteratorCreation =
+      this.trackIteratorCreationPromises.get(trackId);
+    if (ongoingIteratorCreation) {
+      await ongoingIteratorCreation;
+    }
+
+    const iterator = this.trackIterators.get(trackId);
+    if (iterator) {
+      try {
+        await iterator.return?.();
+      } catch (_error) {
+        // Iterator cleanup failed, continue anyway
+      }
+    }
+    this.trackIterators.delete(trackId);
+  }
+
+  private async seekSafe(trackId: number, timeMs: number) {
+    // Get or create track-specific buffer
+    if (!this.trackBuffers.has(trackId)) {
+      await this.createTrackBuffer(trackId);
+    }
+    // biome-ignore lint/style/noNonNullAssertion: we know the map has the key
+    const trackBuffer = this.trackBuffers.get(trackId)!;
+
+    const track = await this.getTrack(trackId);
+
+    // Early validation: check if seek time is outside track bounds
+    // Use consistent precision handling throughout
+    const firstTimestampMs = roundToMilliseconds(
+      (await track.getFirstTimestamp()) * 1000,
+    );
+    let roundedTimeMs = roundToMilliseconds(timeMs);
+
+    // During rapid scrubbing, track.computeDuration() may only return the duration
+    // of currently loaded segments. Only validate against the start time, as the
+    // end time may not be accurate until all segments are loaded.
+    if (roundedTimeMs < firstTimestampMs) {
+      // GRACEFUL HANDLING: During rapid seeking, tasks can complete out of order, causing
+      // the audio buffer to contain segments for a different time range than the seek target.
+      // Only apply graceful adjustment if we have buffer contents that suggest a race condition.
+      // For empty buffers, allow normal seeking to proceed which may load the appropriate segments.
+
+      const bufferContents = trackBuffer.getContents();
+
+      if (bufferContents.length > 0) {
+        // We have loaded segments but they're for a different time range - adjust gracefully
+        timeMs = firstTimestampMs;
+        roundedTimeMs = roundToMilliseconds(timeMs);
+      } else {
+        // Empty buffer - let normal seeking proceed to load appropriate segments
+        // This maintains normal seeking behavior for tests and initial loads
+      }
+    }
+
+    // Note: If seeking beyond currently loaded segments, allow it to proceed
+    // The segment loading logic will handle fetching the needed segments
+    // No logging needed as this is a normal part of seeking behavior
+
+    // Check if we need to reset iterator for seeks outside current buffer range
+    const bufferContents = trackBuffer.getContents();
+    if (bufferContents.length > 0) {
+      const bufferStartMs = roundToMilliseconds(
+        trackBuffer.firstTimestamp * 1000,
+      );
+      const lastSample = bufferContents[bufferContents.length - 1];
+      const bufferEndMs = lastSample
+        ? roundToMilliseconds(
+            (lastSample.timestamp + (lastSample.duration || 0)) * 1000,
+          )
+        : bufferStartMs;
+
+      // If seeking outside current buffer range, reset iterator to load appropriate data
+      if (roundedTimeMs < bufferStartMs || roundedTimeMs > bufferEndMs) {
+        await this.resetIterator(trackId);
+      }
+    }
+
+    const alreadyInBuffer = trackBuffer.find(timeMs);
+    if (alreadyInBuffer) return alreadyInBuffer;
+
+    const iterator = await this.getTrackIterator(trackId);
+    while (true) {
+      const { done, value: decodedSample } = await iterator.next();
+      if (decodedSample) {
+        trackBuffer.push(decodedSample);
+      }
+      const foundSample = trackBuffer.find(timeMs);
+      if (foundSample) {
+        return foundSample;
+      }
+      if (done) {
+        break;
+      }
+    }
+
+    // If no exact sample found and we've reached the end of the track,
+    // check if the seek time is beyond the actual track duration.
+    // If so, return the last available sample instead of throwing an error.
+    const finalBufferContents = trackBuffer.getContents();
+    if (finalBufferContents.length > 0) {
+      const lastSample = finalBufferContents[finalBufferContents.length - 1];
+      const lastSampleEndMs = roundToMilliseconds(
+        ((lastSample?.timestamp || 0) + (lastSample?.duration || 0)) * 1000,
+      );
+
+      // If seeking past the last sample, return the last sample silently
+      if (roundToMilliseconds(timeMs) >= lastSampleEndMs) {
+        return lastSample;
+      }
+    }
+
+    throw new NoSample(
+      `Sample not found for time ${timeMs} in ${track.type} track ${trackId}`,
+    );
+  }
+}

package/src/elements/EFMedia/JitMediaEngine.browsertest.ts

@@ -0,0 +1,165 @@
+import { describe } from "vitest";
+import { test as baseTest } from "../../../test/useMSW.js";
+
+import type { ManifestResponse } from "../../transcoding/types/index.js";
+import { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
+import "../EFVideo.js";
+import type { EFVideo } from "../EFVideo.js";
+import { JitMediaEngine } from "./JitMediaEngine";
+
+const test = baseTest.extend<{
+  emptyManifestResponse: ManifestResponse;
+  urlGenerator: UrlGenerator;
+  manifestUrl: string;
+  mediaEngine: JitMediaEngine;
+  abortSignal: AbortSignal;
+  testUrl: string;
+  host: EFVideo;
+}>({
+  mediaEngine: async ({ manifestUrl, urlGenerator, host }, use: any) => {
+    const engine = await JitMediaEngine.fetch(host, urlGenerator, manifestUrl);
+    await use(engine);
+  },
+  manifestUrl: async ({ urlGenerator, host }, use: any) => {
+    const url = urlGenerator.generateManifestUrl(host.src);
+    await use(url);
+  },
+
+  emptyManifestResponse: async ({}, use: any) => {
+    const emptyResponse: ManifestResponse = {
+      version: "1.0",
+      type: "cmaf",
+      duration: 60,
+      durationMs: 60000,
+      segmentDuration: 4000,
+      baseUrl: "http://api.example.com/",
+      sourceUrl: "http://example.com/video.mp4",
+      audioRenditions: [],
+      videoRenditions: [],
+      endpoints: {
+        initSegment: "http://api.example.com/init/{renditionId}",
+        mediaSegment:
+          "http://api.example.com/segment/{segmentId}/{renditionId}",
+      },
+      jitInfo: {
+        parallelTranscodingSupported: true,
+        expectedTranscodeLatency: 1000,
+        segmentCount: 15,
+      },
+    };
+    await use(emptyResponse);
+  },
+  host: async ({}, use: any) => {
+    const configuration = document.createElement("ef-configuration");
+    // Use integrated proxy server (same host/port as test runner)
+    const apiHost = `${window.location.protocol}//${window.location.host}`;
+    configuration.setAttribute("api-host", apiHost);
+    configuration.apiHost = apiHost;
+    const host = document.createElement("ef-video");
+    configuration.appendChild(host);
+    host.src = "http://web:3000/head-moov-480p.mp4";
+    await use(host);
+  },
+  urlGenerator: async ({}, use: any) => {
+    // UrlGenerator points to integrated proxy server (same host/port as test runner)
+    const apiHost = `${window.location.protocol}//${window.location.host}`;
+    const generator = new UrlGenerator(() => apiHost);
+    await use(generator);
+  },
+
+  abortSignal: async ({}, use: any) => {
+    const signal = new AbortController().signal;
+    await use(signal);
+  },
+  testUrl: async ({}, use: any) => {
+    const url = "http://api.example.com/manifest";
+    await use(url);
+  },
+});
+
+describe("JitMediaEngine", () => {
+  test("provides duration from manifest data", async ({
+    mediaEngine,
+    expect,
+  }) => {
+    expect(mediaEngine.durationMs).toBe(10000);
+  });
+
+  test("provides source URL from manifest data", async ({
+    mediaEngine,
+    host,
+    expect,
+  }) => {
+    expect(mediaEngine.src).toBe(host.src);
+  });
+
+  test("returns audio rendition with correct properties", ({
+    mediaEngine,
+    host,
+    expect,
+  }) => {
+    const audioRendition = mediaEngine.audioRendition;
+
+    expect(audioRendition).toBeDefined();
+    expect(audioRendition!.id).toBe("audio");
+    expect(audioRendition!.trackId).toBeUndefined();
+    expect(audioRendition!.src).toBe(host.src);
+    expect(audioRendition!.segmentDurationMs).toBe(2000);
+  });
+
+  test("returns undefined audio rendition when none available", ({
+    urlGenerator,
+    emptyManifestResponse,
+    host,
+    expect,
+  }) => {
+    const engine = new JitMediaEngine(
+      host,
+      urlGenerator,
+      emptyManifestResponse,
+    );
+
+    expect(engine.audioRendition).toBeUndefined();
+  });
+
+  test("returns video rendition with correct properties", ({
+    mediaEngine,
+    host,
+    expect,
+  }) => {
+    const videoRendition = mediaEngine.videoRendition;
+
+    expect(videoRendition).toBeDefined();
+    expect(videoRendition!.id).toBe("high");
+    expect(videoRendition!.trackId).toBeUndefined();
+    expect(videoRendition!.src).toBe(host.src);
+    expect(videoRendition!.segmentDurationMs).toBe(2000);
+  });
+
+  test("returns undefined video rendition when none available", ({
+    urlGenerator,
+    emptyManifestResponse,
+    host,
+    expect,
+  }) => {
+    const engine = new JitMediaEngine(
+      host,
+      urlGenerator,
+      emptyManifestResponse,
+    );
+
+    expect(engine.videoRendition).toBeUndefined();
+  });
+
+  test("provides templates from manifest endpoints", ({
+    mediaEngine,
+    expect,
+  }) => {
+    expect(mediaEngine.templates).toEqual({
+      initSegment:
+        "http://localhost:63315/api/v1/transcode/{rendition}/init.m4s?url=http%3A%2F%2Fweb%3A3000%2Fhead-moov-480p.mp4",
+      mediaSegment:
+        "http://localhost:63315/api/v1/transcode/{rendition}/{segmentId}.m4s?url=http%3A%2F%2Fweb%3A3000%2Fhead-moov-480p.mp4",
+    });
+  });
+});

package/src/elements/EFMedia/JitMediaEngine.ts

@@ -0,0 +1,166 @@
+import type {
+  AudioRendition,
+  MediaEngine,
+  RenditionId,
+  VideoRendition,
+} from "../../transcoding/types";
+import type { ManifestResponse } from "../../transcoding/types/index.js";
+import type { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
+import type { EFMedia } from "../EFMedia.js";
+import { BaseMediaEngine } from "./BaseMediaEngine";
+
+export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
+  static async fetch(host: EFMedia, urlGenerator: UrlGenerator, url: string) {
+    const response = await host.fetch(url);
+    const data = (await response.json()) as ManifestResponse;
+    return new JitMediaEngine(host, urlGenerator, data);
+  }
+
+  constructor(
+    public host: EFMedia,
+    private urlGenerator: UrlGenerator,
+    private data: ManifestResponse,
+  ) {
+    super();
+  }
+
+  get durationMs() {
+    return this.data.durationMs;
+  }
+
+  get src() {
+    return this.data.sourceUrl;
+  }
+
+  get audioRendition(): AudioRendition | undefined {
+    const rendition = this.data.audioRenditions[0];
+
+    if (!rendition) return undefined;
+    return {
+      id: rendition.id as RenditionId,
+      trackId: undefined,
+      src: this.data.sourceUrl,
+      segmentDurationMs: rendition.segmentDurationMs,
+      segmentDurationsMs: rendition.segmentDurationsMs,
+    };
+  }
+
+  get videoRendition(): VideoRendition | undefined {
+    const rendition = this.data.videoRenditions[0];
+
+    if (!rendition) return undefined;
+    return {
+      id: rendition.id as RenditionId,
+      trackId: undefined,
+      src: this.data.sourceUrl,
+      segmentDurationMs: rendition.segmentDurationMs,
+      segmentDurationsMs: rendition.segmentDurationsMs,
+    };
+  }
+
+  get templates() {
+    return this.data.endpoints;
+  }
+
+  async fetchInitSegment(
+    rendition: { id?: RenditionId; trackId: number | undefined; src: string },
+    signal: AbortSignal,
+  ) {
+    if (!rendition.id) {
+      throw new Error("Rendition ID is required for JIT metadata");
+    }
+    const url = this.urlGenerator.generateSegmentUrl(
+      "init",
+      rendition.id,
+      this,
+    );
+    const response = await this.host.fetch(url, { signal });
+    const arrayBuffer = await response.arrayBuffer();
+    return arrayBuffer;
+  }
+
+  async fetchMediaSegmentImpl(
+    segmentId: number,
+    rendition: { id?: RenditionId; trackId: number | undefined; src: string },
+  ) {
+    if (!rendition.id) {
+      throw new Error("Rendition ID is required for JIT metadata");
+    }
+    const url = this.urlGenerator.generateSegmentUrl(
+      segmentId,
+      rendition.id,
+      this,
+    );
+    return this.fetchMediaCache(url);
+  }
+
+  computeSegmentId(
+    desiredSeekTimeMs: number,
+    rendition: VideoRendition | AudioRendition,
+  ) {
+    // Don't request segments beyond the actual file duration
+    // Note: seeking to exactly durationMs should be allowed (it's the last moment of the file)
+    if (desiredSeekTimeMs > this.durationMs) {
+      return undefined;
+    }
+
+    // Use actual segment durations if available (more accurate)
+    if (
+      rendition.segmentDurationsMs &&
+      rendition.segmentDurationsMs.length > 0
+    ) {
+      let cumulativeTime = 0;
+
+      for (let i = 0; i < rendition.segmentDurationsMs.length; i++) {
+        const segmentDuration = rendition.segmentDurationsMs[i];
+        if (segmentDuration === undefined) {
+          throw new Error("Segment duration is required for JIT metadata");
+        }
+        const segmentStartMs = cumulativeTime;
+        const segmentEndMs = cumulativeTime + segmentDuration;
+
+        // Check if the desired seek time falls within this segment
+        // Special case: for the last segment, include the exact end time
+        const isLastSegment = i === rendition.segmentDurationsMs.length - 1;
+        const includesEndTime =
+          isLastSegment && desiredSeekTimeMs === this.durationMs;
+
+        if (
+          desiredSeekTimeMs >= segmentStartMs &&
+          (desiredSeekTimeMs < segmentEndMs || includesEndTime)
+        ) {
+          return i + 1; // Convert 0-based to 1-based segment ID
+        }
+
+        cumulativeTime += segmentDuration;
+
+        // If we've reached or exceeded file duration, stop
+        if (cumulativeTime >= this.durationMs) {
+          break;
+        }
+      }
+
+      // If we didn't find a segment, return undefined
+      return undefined;
+    }
+
+    // Fall back to fixed duration calculation for backward compatibility
+    if (!rendition.segmentDurationMs) {
+      throw new Error("Segment duration is required for JIT metadata");
+    }
+
+    const segmentIndex = Math.floor(
+      desiredSeekTimeMs / rendition.segmentDurationMs,
+    );
+
+    // Calculate the actual segment start time
+    const segmentStartMs = segmentIndex * rendition.segmentDurationMs;
+
+    // If this segment would start at or beyond file duration, it doesn't exist
+    if (segmentStartMs >= this.durationMs) {
+      return undefined;
+    }
+
+    return segmentIndex + 1; // Convert 0-based to 1-based
+  }
+}