@editframe/elements 0.17.6-beta.0 → 0.18.3-beta.0

package/src/elements/EFMedia/BufferedSeekingInput.ts

@@ -0,0 +1,267 @@
+import {
+  AudioSampleSink,
+  BufferSource,
+  Input,
+  MP4,
+  VideoSampleSink,
+} from "mediabunny";
+import { type MediaSample, SampleBuffer } from "../SampleBuffer";
+
+interface BufferedSeekingInputOptions {
+  videoBufferSize?: number;
+  audioBufferSize?: number;
+  /**
+   * FFmpeg start_time offset in milliseconds from the processed video.
+   * Applied during seeking to correct for timing shifts introduced by FFmpeg processing.
+   */
+  startTimeOffsetMs?: number;
+}
+
+const defaultOptions: BufferedSeekingInputOptions = {
+  videoBufferSize: 30,
+  audioBufferSize: 100,
+  startTimeOffsetMs: 0,
+};
+
+export class NoSample extends RangeError {}
+
+export class BufferedSeekingInput {
+  private input: Input;
+  private trackIterators: Map<number, AsyncIterator<MediaSample>> = new Map();
+  private trackBuffers: Map<number, SampleBuffer> = new Map();
+  private options: BufferedSeekingInputOptions;
+  // Separate locks for different operation types to prevent unnecessary blocking
+  private trackIteratorCreationPromises: Map<number, Promise<any>> = new Map();
+  private trackSeekPromises: Map<number, Promise<any>> = new Map();
+
+  /**
+   * FFmpeg start_time offset in milliseconds from the processed video.
+   * Applied during seeking to correct for timing shifts introduced by FFmpeg processing.
+   */
+  private readonly startTimeOffsetMs: number;
+
+  constructor(arrayBuffer: ArrayBuffer, options?: BufferedSeekingInputOptions) {
+    const bufferSource = new BufferSource(arrayBuffer);
+    const input = new Input({
+      source: bufferSource,
+      formats: [MP4],
+    });
+    this.input = input;
+    this.options = { ...defaultOptions, ...options };
+    this.startTimeOffsetMs = this.options.startTimeOffsetMs ?? 0;
+  }
+
+  // Buffer inspection API for testing
+  getBufferSize(trackId: number): number {
+    const buffer = this.trackBuffers.get(trackId);
+    return buffer ? buffer.length : 0;
+  }
+
+  getBufferContents(trackId: number): readonly MediaSample[] {
+    const buffer = this.trackBuffers.get(trackId);
+    return buffer ? Object.freeze([...buffer.getContents()]) : [];
+  }
+
+  getBufferTimestamps(trackId: number): number[] {
+    const contents = this.getBufferContents(trackId);
+    return contents.map((sample) => sample.timestamp || 0);
+  }
+
+  clearBuffer(trackId: number): void {
+    const buffer = this.trackBuffers.get(trackId);
+    if (buffer) {
+      buffer.clear();
+    }
+  }
+
+  computeDuration() {
+    return this.input.computeDuration();
+  }
+
+  async getTrack(trackId: number) {
+    const tracks = await this.input.getTracks();
+    const track = tracks.find((track) => track.id === trackId);
+    if (!track) {
+      throw new Error(`Track ${trackId} not found`);
+    }
+    return track;
+  }
+
+  async getAudioTrack(trackId: number) {
+    const tracks = await this.input.getAudioTracks();
+    const track = tracks.find(
+      (track) => track.id === trackId && track.type === "audio",
+    );
+    if (!track) {
+      throw new Error(`Track ${trackId} not found`);
+    }
+    return track;
+  }
+
+  async getVideoTrack(trackId: number) {
+    const tracks = await this.input.getVideoTracks();
+    const track = tracks.find(
+      (track) => track.id === trackId && track.type === "video",
+    );
+    if (!track) {
+      throw new Error(`Track ${trackId} not found`);
+    }
+    return track;
+  }
+
+  async getFirstVideoTrack() {
+    const tracks = await this.input.getVideoTracks();
+    return tracks[0];
+  }
+
+  async getFirstAudioTrack() {
+    const tracks = await this.input.getAudioTracks();
+    return tracks[0];
+  }
+
+  async getTrackIterator(trackId: number) {
+    if (this.trackIterators.has(trackId)) {
+      // biome-ignore lint/style/noNonNullAssertion: we know the map has the key
+      return this.trackIterators.get(trackId)!;
+    }
+
+    // Serialize iterator creation per track (but don't block seeks)
+    const existingIteratorCreation =
+      this.trackIteratorCreationPromises.get(trackId);
+    if (existingIteratorCreation) {
+      await existingIteratorCreation;
+      // Check again after waiting - another operation might have created it
+      if (this.trackIterators.has(trackId)) {
+        // biome-ignore lint/style/noNonNullAssertion: we know the map has the key
+        return this.trackIterators.get(trackId)!;
+      }
+    }
+
+    const creationPromise = this.createIteratorSafe(trackId);
+    this.trackIteratorCreationPromises.set(trackId, creationPromise);
+
+    try {
+      const iterator = await creationPromise;
+      return iterator;
+    } finally {
+      this.trackIteratorCreationPromises.delete(trackId);
+    }
+  }
+
+  private async createIteratorSafe(trackId: number) {
+    const track = await this.getTrack(trackId);
+    if (track.type === "audio") {
+      const track = await this.getAudioTrack(trackId);
+      const sampleSink = new AudioSampleSink(track);
+      const iterator = sampleSink.samples();
+      this.trackIterators.set(trackId, iterator);
+      return iterator;
+    }
+    {
+      const track = await this.getVideoTrack(trackId);
+      const sampleSink = new VideoSampleSink(track);
+      const iterator = sampleSink.samples();
+      this.trackIterators.set(trackId, iterator);
+      return iterator;
+    }
+  }
+
+  async createTrackBuffer(trackId: number) {
+    const track = await this.getTrack(trackId);
+    if (track.type === "audio") {
+      const bufferSize = this.options.audioBufferSize;
+      this.trackBuffers.set(trackId, new SampleBuffer(bufferSize));
+    } else {
+      const bufferSize = this.options.videoBufferSize;
+      this.trackBuffers.set(trackId, new SampleBuffer(bufferSize));
+    }
+  }
+
+  async seek(trackId: number, timeMs: number) {
+    // Apply the start time offset to deal with files that don't start on the zero
+    const correctedTimeMs = timeMs + this.startTimeOffsetMs;
+
+    // Serialize seek operations per track (but don't block iterator creation)
+    const existingSeek = this.trackSeekPromises.get(trackId);
+    if (existingSeek) {
+      await existingSeek;
+    }
+
+    const seekPromise = this.seekSafe(trackId, correctedTimeMs);
+    this.trackSeekPromises.set(trackId, seekPromise);
+
+    try {
+      return await seekPromise;
+    } finally {
+      this.trackSeekPromises.delete(trackId);
+    }
+  }
+
+  private async resetIterator(trackId: number) {
+    const trackBuffer = this.trackBuffers.get(trackId);
+    trackBuffer?.clear();
+    // Clean up iterator safely - wait for any ongoing iterator creation
+    const ongoingIteratorCreation =
+      this.trackIteratorCreationPromises.get(trackId);
+    if (ongoingIteratorCreation) {
+      await ongoingIteratorCreation;
+    }
+
+    const iterator = this.trackIterators.get(trackId);
+    if (iterator) {
+      try {
+        await iterator.return?.();
+      } catch (_error) {
+        // Iterator cleanup failed, continue anyway
+      }
+    }
+    this.trackIterators.delete(trackId);
+  }
+
+  private async seekSafe(trackId: number, timeMs: number) {
+    // Get or create track-specific buffer
+    if (!this.trackBuffers.has(trackId)) {
+      await this.createTrackBuffer(trackId);
+    }
+    // biome-ignore lint/style/noNonNullAssertion: we know the map has the key
+    const trackBuffer = this.trackBuffers.get(trackId)!;
+
+    if (timeMs < trackBuffer.firstTimestamp * 1000) {
+      await this.resetIterator(trackId);
+    }
+
+    const alreadyInBuffer = trackBuffer.find(timeMs);
+    const track = await this.getTrack(trackId);
+
+    // Early validation: check if seek time is outside track bounds
+    const firstTimestampMs = (await track.getFirstTimestamp()) * 1000;
+    const lastSampleEndMs = (await track.computeDuration()) * 1000; // computeDuration returns end time of last sample
+
+    if (timeMs < firstTimestampMs || timeMs >= lastSampleEndMs) {
+      throw new NoSample(
+        `Seek time ${timeMs}ms is outside track range [${firstTimestampMs}ms, ${lastSampleEndMs}ms]`,
+      );
+    }
+
+    if (alreadyInBuffer) return alreadyInBuffer;
+
+    const iterator = await this.getTrackIterator(trackId);
+    while (true) {
+      const { done, value: decodedSample } = await iterator.next();
+      if (decodedSample) {
+        trackBuffer.push(decodedSample);
+      }
+      const foundSample = trackBuffer.find(timeMs);
+      if (foundSample) {
+        return foundSample;
+      }
+      if (done) {
+        break;
+      }
+    }
+
+    throw new NoSample(
+      `Sample not found for time ${timeMs} in ${track.type} track ${trackId}`,
+    );
+  }
+}

package/src/elements/EFMedia/JitMediaEngine.browsertest.ts

@@ -0,0 +1,165 @@
+import { describe } from "vitest";
+import { test as baseTest } from "../../../test/useMSW.js";
+
+import type { ManifestResponse } from "../../transcoding/types/index.js";
+import { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
+import "../EFVideo.js";
+import type { EFVideo } from "../EFVideo.js";
+import { JitMediaEngine } from "./JitMediaEngine";
+
+const test = baseTest.extend<{
+  emptyManifestResponse: ManifestResponse;
+  urlGenerator: UrlGenerator;
+  manifestUrl: string;
+  mediaEngine: JitMediaEngine;
+  abortSignal: AbortSignal;
+  testUrl: string;
+  host: EFVideo;
+}>({
+  mediaEngine: async ({ manifestUrl, urlGenerator, host }, use: any) => {
+    const engine = await JitMediaEngine.fetch(host, urlGenerator, manifestUrl);
+    await use(engine);
+  },
+  manifestUrl: async ({ urlGenerator, host }, use: any) => {
+    const url = urlGenerator.generateManifestUrl(host.src);
+    await use(url);
+  },
+
+  emptyManifestResponse: async ({}, use: any) => {
+    const emptyResponse: ManifestResponse = {
+      version: "1.0",
+      type: "cmaf",
+      duration: 60,
+      durationMs: 60000,
+      segmentDuration: 4000,
+      baseUrl: "http://api.example.com/",
+      sourceUrl: "http://example.com/video.mp4",
+      audioRenditions: [],
+      videoRenditions: [],
+      endpoints: {
+        initSegment: "http://api.example.com/init/{renditionId}",
+        mediaSegment:
+          "http://api.example.com/segment/{segmentId}/{renditionId}",
+      },
+      jitInfo: {
+        parallelTranscodingSupported: true,
+        expectedTranscodeLatency: 1000,
+        segmentCount: 15,
+      },
+    };
+    await use(emptyResponse);
+  },
+  host: async ({}, use: any) => {
+    const configuration = document.createElement("ef-configuration");
+    // Use integrated proxy server (same host/port as test runner)
+    const apiHost = `${window.location.protocol}//${window.location.host}`;
+    configuration.setAttribute("api-host", apiHost);
+    configuration.apiHost = apiHost;
+    const host = document.createElement("ef-video");
+    configuration.appendChild(host);
+    host.src = "http://web:3000/head-moov-480p.mp4";
+    await use(host);
+  },
+  urlGenerator: async ({}, use: any) => {
+    // UrlGenerator points to integrated proxy server (same host/port as test runner)
+    const apiHost = `${window.location.protocol}//${window.location.host}`;
+    const generator = new UrlGenerator(() => apiHost);
+    await use(generator);
+  },
+
+  abortSignal: async ({}, use: any) => {
+    const signal = new AbortController().signal;
+    await use(signal);
+  },
+  testUrl: async ({}, use: any) => {
+    const url = "http://api.example.com/manifest";
+    await use(url);
+  },
+});
+
+describe("JitMediaEngine", () => {
+  test("provides duration from manifest data", async ({
+    mediaEngine,
+    expect,
+  }) => {
+    expect(mediaEngine.durationMs).toBe(10000);
+  });
+
+  test("provides source URL from manifest data", async ({
+    mediaEngine,
+    host,
+    expect,
+  }) => {
+    expect(mediaEngine.src).toBe(host.src);
+  });
+
+  test("returns audio rendition with correct properties", ({
+    mediaEngine,
+    host,
+    expect,
+  }) => {
+    const audioRendition = mediaEngine.audioRendition;
+
+    expect(audioRendition).toBeDefined();
+    expect(audioRendition!.id).toBe("audio");
+    expect(audioRendition!.trackId).toBeUndefined();
+    expect(audioRendition!.src).toBe(host.src);
+    expect(audioRendition!.segmentDurationMs).toBe(2000);
+  });
+
+  test("returns undefined audio rendition when none available", ({
+    urlGenerator,
+    emptyManifestResponse,
+    host,
+    expect,
+  }) => {
+    const engine = new JitMediaEngine(
+      host,
+      urlGenerator,
+      emptyManifestResponse,
+    );
+
+    expect(engine.audioRendition).toBeUndefined();
+  });
+
+  test("returns video rendition with correct properties", ({
+    mediaEngine,
+    host,
+    expect,
+  }) => {
+    const videoRendition = mediaEngine.videoRendition;
+
+    expect(videoRendition).toBeDefined();
+    expect(videoRendition!.id).toBe("high");
+    expect(videoRendition!.trackId).toBeUndefined();
+    expect(videoRendition!.src).toBe(host.src);
+    expect(videoRendition!.segmentDurationMs).toBe(2000);
+  });
+
+  test("returns undefined video rendition when none available", ({
+    urlGenerator,
+    emptyManifestResponse,
+    host,
+    expect,
+  }) => {
+    const engine = new JitMediaEngine(
+      host,
+      urlGenerator,
+      emptyManifestResponse,
+    );
+
+    expect(engine.videoRendition).toBeUndefined();
+  });
+
+  test("provides templates from manifest endpoints", ({
+    mediaEngine,
+    expect,
+  }) => {
+    expect(mediaEngine.templates).toEqual({
+      initSegment:
+        "http://localhost:63315/api/v1/transcode/{rendition}/init.m4s?url=http%3A%2F%2Fweb%3A3000%2Fhead-moov-480p.mp4",
+      mediaSegment:
+        "http://localhost:63315/api/v1/transcode/{rendition}/{segmentId}.m4s?url=http%3A%2F%2Fweb%3A3000%2Fhead-moov-480p.mp4",
+    });
+  });
+});

package/src/elements/EFMedia/JitMediaEngine.ts

@@ -0,0 +1,110 @@
+import type {
+  AudioRendition,
+  MediaEngine,
+  RenditionId,
+  VideoRendition,
+} from "../../transcoding/types";
+import type { ManifestResponse } from "../../transcoding/types/index.js";
+import type { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
+import type { EFMedia } from "../EFMedia.js";
+import { BaseMediaEngine } from "./BaseMediaEngine";
+
+export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
+  static async fetch(host: EFMedia, urlGenerator: UrlGenerator, url: string) {
+    const response = await host.fetch(url);
+    const data = (await response.json()) as ManifestResponse;
+    return new JitMediaEngine(host, urlGenerator, data);
+  }
+
+  constructor(
+    public host: EFMedia,
+    private urlGenerator: UrlGenerator,
+    private data: ManifestResponse,
+  ) {
+    super();
+  }
+
+  get durationMs() {
+    return this.data.durationMs;
+  }
+
+  get src() {
+    return this.data.sourceUrl;
+  }
+
+  get audioRendition(): AudioRendition | undefined {
+    const rendition = this.data.audioRenditions[0];
+
+    if (!rendition) return undefined;
+    return {
+      id: rendition.id as RenditionId,
+      trackId: undefined,
+      src: this.data.sourceUrl,
+      segmentDurationMs: rendition.segmentDurationMs,
+    };
+  }
+
+  get videoRendition(): VideoRendition | undefined {
+    const rendition = this.data.videoRenditions[0];
+
+    if (!rendition) return undefined;
+    return {
+      id: rendition.id as RenditionId,
+      trackId: undefined,
+      src: this.data.sourceUrl,
+      segmentDurationMs: rendition.segmentDurationMs,
+    };
+  }
+
+  get templates() {
+    return this.data.endpoints;
+  }
+
+  async fetchInitSegment(
+    rendition: { id?: RenditionId; trackId: number | undefined; src: string },
+    signal: AbortSignal,
+  ) {
+    if (!rendition.id) {
+      throw new Error("Rendition ID is required for JIT metadata");
+    }
+    const url = this.urlGenerator.generateSegmentUrl(
+      "init",
+      rendition.id,
+      this,
+    );
+    const response = await this.host.fetch(url, { signal });
+    const arrayBuffer = await response.arrayBuffer();
+    return arrayBuffer;
+  }
+
+  async fetchMediaSegmentImpl(
+    segmentId: number,
+    rendition: { id?: RenditionId; trackId: number | undefined; src: string },
+  ) {
+    if (!rendition.id) {
+      throw new Error("Rendition ID is required for JIT metadata");
+    }
+    const url = this.urlGenerator.generateSegmentUrl(
+      segmentId,
+      rendition.id,
+      this,
+    );
+    return this.fetchMediaCache(url);
+  }
+
+  computeSegmentId(
+    desiredSeekTimeMs: number,
+    rendition: VideoRendition | AudioRendition,
+  ) {
+    if (!rendition.segmentDurationMs) {
+      throw new Error("Segment duration is required for JIT metadata");
+    }
+    const segmentIndex = Math.floor(
+      desiredSeekTimeMs / rendition.segmentDurationMs,
+    );
+    if (segmentIndex * rendition.segmentDurationMs >= this.durationMs) {
+      return segmentIndex; // Return 1-based index (segmentIndex is already 0-based, so segmentIndex for the last segment is what we want)
+    }
+    return segmentIndex + 1; // Convert 0-based to 1-based
+  }
+}