@twick/2d 0.11.0

package/src/lib/utils/video/mp4-parser-manager.ts

@@ -0,0 +1,72 @@
+import {Mp4Parser} from './parser';
+
+// List of VideoFrameExtractors
+const videoFrameExtractors = new Map<string, Mp4Parser>();
+
+export async function dropExtractor(id: string, filePath: string) {
+  const extractorId = filePath + '-' + id;
+  const extractor = videoFrameExtractors.get(extractorId);
+  if (extractor) {
+    extractor.close();
+    videoFrameExtractors.delete(extractorId);
+  }
+}
+
+export async function getFrame(
+  id: string,
+  filePath: string,
+  time: number,
+  fps: number,
+) {
+  // Check if we already have a VideoFrameExtractor for this video
+  const extractorId = filePath + '-' + id;
+  let extractor = videoFrameExtractors.get(extractorId);
+
+  const frameDuration = 1 / fps;
+
+  /**
+   * Sometimes, HTMLVideoElement.duration is not accurate, which can lead to the
+   * requested time being greater than the duration.
+   * To prevent this, we clamp the time to the duration reported by the extractor.
+   */
+  const duration = extractor?.getDuration();
+  if (duration && time > duration) {
+    time = duration;
+  }
+
+  const isOldFrame =
+    extractor && Math.abs(time - extractor.getLastTime()) < frameDuration / 2;
+
+  // If time has not changed, return the last frame
+  if (extractor && isOldFrame) {
+    const lastFrame = extractor.getLastFrame();
+    if (!lastFrame) {
+      throw new Error('No last frame');
+    }
+
+    return lastFrame;
+  }
+
+  // If the video has skipped back we need to create a new extractor
+  if (extractor && time + frameDuration < extractor.getTime()) {
+    extractor.close();
+    videoFrameExtractors.delete(extractorId);
+    extractor = undefined;
+  }
+
+  // If the video has skipped forward we need to create a new extractor
+  if (extractor && time > extractor.getTime() + frameDuration * 1.5) {
+    extractor.close();
+    videoFrameExtractors.delete(extractorId);
+    extractor = undefined;
+  }
+
+  if (!extractor) {
+    extractor = new Mp4Parser(filePath, fps, time);
+    await extractor.start();
+    videoFrameExtractors.set(extractorId, extractor);
+  }
+
+  // Go to the frame that is closest to the requested time
+  return extractor.getNextFrame();
+}

package/src/lib/utils/video/parser/index.ts

@@ -0,0 +1 @@
+export * from './parser';

package/src/lib/utils/video/parser/parser.ts

@@ -0,0 +1,257 @@
+import {createFile} from 'mp4box';
+import {FrameSampler} from './sampler';
+import {Segment} from './segment';
+import {MP4FileSink} from './sink';
+import type {Edit} from './utils';
+import {description, mp4BoxEditToEdit} from './utils';
+
+/**
+ * Loads the file at the given URI until it finds the moov box.
+ * Once found, it calls `setConfig` with the video configuration.
+ * @param uri - The URI of the video file.
+ * @param setConfig - Callback to set the video configuration.
+ * @returns
+ */
+async function getFileInfo(uri: string) {
+  return new Promise<{
+    file: any;
+    edits: Edit[];
+    config: VideoDecoderConfig;
+  }>((res, rej) => {
+    const file = createFile();
+    let found = false;
+    const controller = new AbortController();
+
+    file.onReady = (info: any) => {
+      found = true;
+      controller.abort();
+
+      const track = info.videoTracks[0];
+
+      const config = {
+        // Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
+        // they only support `vp8`.
+        codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
+        codedHeight: track.video.height,
+        codedWidth: track.video.width,
+        description: description(file, track),
+      };
+
+      const defaultEdit: Omit<Edit, 'fps'> = {
+        mediaTime: 0,
+        segmentDuration: track.duration,
+        mediaRateInteger: 1,
+        mediaRateFraction: 0,
+      };
+
+      const editsFromVideo: Omit<Edit, 'fps'>[] = track.edits?.map(
+        (edit: any) => mp4BoxEditToEdit(edit),
+      );
+
+      // If there are no entries, use the default edit
+      const editsWithoutFps = editsFromVideo?.length
+        ? editsFromVideo
+        : [defaultEdit];
+
+      // Calculate FPS for each segment
+      const edits = editsWithoutFps.map(edit => {
+        const trackDurationInSec = track.duration / track.timescale;
+        const segmentDurationInSec =
+          edit.segmentDuration / track.movie_timescale;
+        const segmentFrames =
+          track.nb_samples * (segmentDurationInSec / trackDurationInSec);
+        const mediaRate =
+          edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+        const fps = (segmentFrames / segmentDurationInSec) * mediaRate;
+        return {...edit, fps};
+      });
+
+      file.setExtractionOptions(track.id);
+      file.start();
+
+      res({file, edits, config});
+    };
+
+    return fetch(uri, {signal: controller.signal}).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {});
+
+      while (!found) {
+        await reader.read().then(({done, value}) => {
+          if (done) {
+            file.flush();
+            controller.abort();
+            rej('Could not find moov');
+            return;
+          }
+
+          sink.write(value);
+        });
+      }
+    });
+  });
+}
+
+export class Mp4Parser {
+  private uri: string;
+  private file: any;
+  private edits: Edit[] = [];
+  private decoderConfig?: VideoDecoderConfig;
+
+  private startTime: number;
+  private targetFps: number;
+
+  private nextSegment = 0;
+  private sampler?: FrameSampler;
+
+  public constructor(uri: string, targetFps: number, startTime: number) {
+    this.uri = uri;
+    this.targetFps = targetFps;
+    this.startTime = startTime;
+  }
+
+  public async start() {
+    const {file, edits, config} = await getFileInfo(this.uri);
+    this.file = file;
+    this.edits = edits;
+    this.decoderConfig = config;
+  }
+
+  public async getNextFrame() {
+    // Start the first segment
+    if (!this.sampler) {
+      // Skip segments until the start time
+      let startTimeWithinSegment = this.startTime;
+      while (this.nextSegment < this.edits.length) {
+        const segmentDurationInSeconds = this.getSecondDurationOfSegment(
+          this.edits[this.nextSegment],
+        );
+        if (startTimeWithinSegment < segmentDurationInSeconds) {
+          break;
+        }
+
+        startTimeWithinSegment -= segmentDurationInSeconds;
+        this.nextSegment++;
+      }
+
+      // The timestamp is outside of the video
+      if (this.nextSegment >= this.edits.length) {
+        throw new Error(
+          `Timestamp ${this.startTime} is outside of the video, max timestamp is ${this.getDuration()}`,
+        );
+      }
+
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        startTimeWithinSegment,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        0,
+      );
+      this.nextSegment++;
+    }
+
+    // Try to get the next frame
+    let frame = await this.sampler.getNextFrame();
+
+    // If there are no more frames in the current segment, start the next segment
+    while (!frame && this.nextSegment < this.edits.length) {
+      this.sampler.getSegment().close();
+      this.sampler.getLastFrame()?.close();
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        0,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        this.sampler.getSum(), // Carry over the sum from the previous segment
+      );
+      this.nextSegment++;
+
+      frame = await this.sampler.getNextFrame();
+    }
+
+    // If there are no more frames, return the last frame
+    if (!frame) {
+      frame = this.sampler.getLastFrame();
+
+      // We can close the segment
+      await this.sampler.getSegment().close();
+
+      // If we still don't have a frame, throw an error
+      if (!frame) {
+        throw new Error('There are no frames in the video.');
+      }
+    }
+
+    return frame;
+  }
+
+  private getSecondDurationOfSegment(edit: Edit) {
+    const mediaRate = edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+    const duration =
+      edit.segmentDuration / this.file.getInfo().videoTracks[0].movie_timescale;
+    return duration / mediaRate;
+  }
+
+  public getDuration() {
+    return this.edits.reduce(
+      (acc, edit) => acc + this.getSecondDurationOfSegment(edit),
+      0,
+    );
+  }
+
+  private getTimeSubtractingFrames(frames: number) {
+    const durationOfPastSegmentsInSeconds = this.edits
+      .slice(0, this.nextSegment - 1)
+      .reduce((acc, edit) => acc + this.getSecondDurationOfSegment(edit), 0);
+
+    if (!this.sampler) {
+      throw new Error('No current segment');
+    }
+
+    const currentSegmentStartTime = this.sampler.getSegment().getStartTime();
+    const samplerTime = this.sampler.getTime(frames);
+
+    return (
+      durationOfPastSegmentsInSeconds + currentSegmentStartTime + samplerTime
+    );
+  }
+
+  public getTime() {
+    return this.getTimeSubtractingFrames(0);
+  }
+
+  public getLastTime() {
+    return this.getTimeSubtractingFrames(1);
+  }
+
+  public getLastFrame() {
+    return this.sampler?.getLastFrame();
+  }
+
+  public close() {
+    this.sampler?.getSegment().close();
+    this.sampler?.getLastFrame()?.close();
+    this.file.flush();
+  }
+}

package/src/lib/utils/video/parser/sampler.ts

@@ -0,0 +1,72 @@
+import type {Segment} from './segment';
+
+export class FrameSampler {
+  private framesRequested = 0;
+  private lastFrame?: VideoFrame;
+
+  public constructor(
+    private segment: Segment,
+    private sourceFps: number,
+    private targetFps: number,
+    private sum: number = 0,
+  ) {}
+
+  public async getNextFrame() {
+    const samplingRate = this.sourceFps / this.targetFps;
+    this.sum += samplingRate;
+    this.framesRequested += 1;
+
+    // Return last frame if we haven't progressed enough to get a new frame.
+    if (this.sum < 1 && this.lastFrame) {
+      return this.lastFrame;
+    }
+
+    // Burn frames if we have too many.
+    while (this.sum >= 2) {
+      // Burn frame
+      const frame = await this.segment.getNextFrame();
+
+      // If there are no more frames, return
+      if (!frame) {
+        return;
+      }
+
+      frame.close();
+      this.sum -= 1;
+    }
+
+    // Get a new frame.
+    if (this.sum >= 1 || !this.lastFrame) {
+      this.sum -= 1;
+      const frame = await this.segment.getNextFrame();
+
+      // If there are no more frames, return
+      if (!frame) {
+        return;
+      }
+
+      this.lastFrame?.close();
+      this.lastFrame = frame;
+      return frame;
+    }
+
+    // One of the three if statements above is always true.
+    throw new Error('Unreachable code');
+  }
+
+  public getLastFrame() {
+    return this.lastFrame;
+  }
+
+  public getSum() {
+    return this.sum;
+  }
+
+  public getSegment() {
+    return this.segment;
+  }
+
+  public getTime(framesToSubtract: number) {
+    return (this.framesRequested - framesToSubtract) / this.targetFps;
+  }
+}

package/src/lib/utils/video/parser/segment.ts

@@ -0,0 +1,302 @@
+import {MP4FileSink} from './sink';
+import type {Edit} from './utils';
+
+const MAX_DECODE_QUEUE_SIZE = 30;
+
+export class Segment {
+  private done: boolean = false;
+  private currentFramePastSegmentEndTime: boolean = false;
+  private abortController = new AbortController();
+  private uri: string;
+
+  private file: any;
+  private edit: Edit;
+
+  private responseFinished: boolean = false;
+  private decoder: VideoDecoder;
+
+  private framesProcessed = 0;
+  private startTime: number;
+  private framesDue = 0;
+  private frameBuffer: VideoFrame[] = [];
+  private encodedChunkQueue: EncodedVideoChunk[] = [];
+
+  private readMore: () => Promise<void> = async () => {};
+
+  public constructor(
+    uri: string,
+    file: any,
+    edits: Edit[],
+    currentSegment: number,
+    startTime: number = 0,
+  ) {
+    this.uri = uri;
+
+    this.file = file;
+    this.file.onSamples = this.onSamples.bind(this);
+    this.edit = edits[currentSegment];
+
+    this.startTime = startTime;
+
+    // Check how many frames we skipped because of the startTime and add them to the frames processed.
+    const framesSkipped = Math.floor(this.startTime * this.edit.fps);
+    this.framesProcessed += framesSkipped;
+
+    this.decoder = new VideoDecoder({
+      output: this.onFrame.bind(this),
+      error(e) {
+        console.error(e);
+      },
+    });
+  }
+
+  public async start(decoderConfig: VideoDecoderConfig) {
+    // If this is an empty edit, we just need to fill the buffer with empty frames.
+    if (this.edit.mediaTime === -1) {
+      this.done = true;
+      this.responseFinished = true;
+      this.decoder.close();
+
+      const segmentDurationInSeconds =
+        this.edit.segmentDuration /
+        this.file.getInfo().videoTracks[0].movie_timescale;
+      const framesToFill = Math.ceil(segmentDurationInSeconds * this.edit.fps);
+
+      const height = this.file.getInfo().videoTracks[0].track_height;
+      const width = this.file.getInfo().videoTracks[0].track_width;
+
+      const bufferSize = height * width * 4;
+      const buffer = new ArrayBuffer(bufferSize);
+      const uint8Array = new Uint8Array(buffer);
+
+      // Make the frame black
+      uint8Array.fill(0);
+
+      this.frameBuffer = Array.from({length: framesToFill}, () => {
+        return new VideoFrame(uint8Array, {
+          timestamp: 0,
+          duration: 1 / this.edit.fps,
+          codedHeight: height,
+          codedWidth: width,
+          format: 'BGRA',
+        });
+      });
+
+      return;
+    }
+
+    this.decoder.configure(decoderConfig);
+    const videoTrack = this.file.getInfo().videoTracks[0];
+    const trak = this.file.getTrackById(videoTrack.id);
+
+    const mediaTimeInTimescale = this.edit.mediaTime;
+    const startTimeInTimescale = this.startTime * videoTrack.timescale;
+    const seekTimeInTimescale = mediaTimeInTimescale + startTimeInTimescale;
+
+    const seekTimeInSec = seekTimeInTimescale / videoTrack.timescale;
+    const seekInfo = this.file.seekTrack(seekTimeInSec, true, trak);
+    this.readMore = await this.startStreamingAtOffset(
+      this.file,
+      this.uri,
+      seekInfo.offset,
+    );
+  }
+
+  /**
+   * Starts streaming the video at the given URI from the given offset.
+   * @param file - MP4Box file. Needs to be created and configured before calling this function.
+   * @param uri - URI of the video file.
+   * @param offset - Offset to start streaming from.
+   * @returns - A function to read more data from the response.
+   */
+  private async startStreamingAtOffset(file: any, uri: string, offset: number) {
+    return fetch(uri, {
+      headers: {
+        /* eslint-disable-next-line @typescript-eslint/naming-convention */
+        Range: `bytes=${offset}-`,
+      },
+      signal: this.abortController.signal,
+    }).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {}, offset);
+
+      return async () => {
+        return reader.read().then(({done, value}) => {
+          // Request is done.
+          if (done) {
+            this.responseFinished = true;
+            this.abortController.abort();
+            sink.close();
+            return;
+          }
+
+          sink.write(value);
+        });
+      };
+    });
+  }
+
+  /**
+   * Called when samples are available on the MP4 file.
+   * Sends chunks to the decoder.
+   */
+  private onSamples(_unused1: any, _unused2: any, samples: any) {
+    for (const sample of samples) {
+      const chunk = new EncodedVideoChunk({
+        type: sample.is_sync ? 'key' : 'delta',
+        timestamp: (1e6 * sample.cts) / sample.timescale,
+        duration: (1e6 * sample.duration) / sample.timescale,
+        data: sample.data,
+      });
+      this.framesDue++;
+      this.encodedChunkQueue.push(chunk);
+
+      const videoTrack = this.file.getInfo().videoTracks[0];
+      const trak = this.file.getTrackById(videoTrack.id);
+      this.file.releaseSample(trak, sample.number);
+    }
+  }
+
+  private async decodeChunks() {
+    while (
+      this.encodedChunkQueue.length > 0 &&
+      this.decoder.decodeQueueSize < MAX_DECODE_QUEUE_SIZE
+    ) {
+      const chunk = this.encodedChunkQueue.shift();
+      if (chunk) {
+        this.decoder.decode(chunk);
+      }
+    }
+    // When edit is empty, we cannot call decoder.flush() because decoder was already closed
+    if (this.done) {
+      this.currentFramePastSegmentEndTime = true;
+      return;
+    }
+    if (this.responseFinished && this.encodedChunkQueue.length === 0) {
+      await this.flushDecoderWithRetry();
+      this.currentFramePastSegmentEndTime = true;
+      return;
+    }
+  }
+
+  private async flushDecoderWithRetry(maxRetries = 3, timeoutMs = 2000) {
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      // Sometimes decoder.flush does not resolve even though the queue size is zero
+      if (this.decoder.decodeQueueSize === 0) {
+        return;
+      }
+      try {
+        await Promise.race([
+          this.decoder.flush(),
+          new Promise((_, reject) =>
+            setTimeout(() => reject(new Error('Flush timeout')), timeoutMs),
+          ),
+        ]);
+      } catch (error: any) {
+        if (attempt === maxRetries) {
+          throw error;
+        }
+      }
+    }
+  }
+
+  /**
+   * Called when the decoder has a frame ready.
+   * Pushes the frame to the buffer so it can be consumed.
+   */
+  private async onFrame(frame: VideoFrame) {
+    this.framesDue--;
+
+    // If the frame comes before the seek time, close it.
+    const mediaTimeInSec =
+      this.edit.mediaTime / this.file.getInfo().videoTracks[0].timescale;
+    const seekTimeInSec = this.startTime + mediaTimeInSec;
+    const frameTimeInSec = frame.timestamp / 1e6;
+    if (frameTimeInSec < seekTimeInSec) {
+      frame.close();
+      return;
+    }
+
+    // Check if we are past the segment duration.
+    const segmentDurationInSec =
+      this.edit.segmentDuration /
+      this.file.getInfo().videoTracks[0].movie_timescale;
+    const segmentEndTime = mediaTimeInSec + segmentDurationInSec;
+    if (frameTimeInSec > segmentEndTime) {
+      frame.close();
+      this.done = true;
+      await this.decoder.flush();
+      return;
+    }
+
+    this.frameBuffer.push(frame);
+  }
+
+  private async populateBuffer() {
+    // Fetch more frames if we don't have any.
+    while (
+      this.frameBuffer.length === 0 &&
+      !this.currentFramePastSegmentEndTime
+    ) {
+      if (!this.responseFinished) {
+        await this.readMore();
+      }
+      await this.decodeChunks();
+      await new Promise(res => setTimeout(res, 0));
+    }
+
+    // Wait for decoder if there are frames due.
+    if (this.frameBuffer.length === 0 && this.framesDue > 0) {
+      let maxIterations = 200;
+      while (this.frameBuffer.length === 0) {
+        await new Promise(res => setTimeout(res, 10));
+        maxIterations--;
+
+        if (this.done) {
+          return;
+        }
+
+        if (maxIterations === 0) {
+          return; // TODO: investigate further, shouldn't this be an error?
+        }
+      }
+    }
+  }
+
+  public async getNextFrame() {
+    await this.populateBuffer();
+    const frame = this.frameBuffer.shift();
+    if (frame) {
+      this.framesProcessed++;
+    }
+    return frame;
+  }
+
+  /**
+   * Called when we are done with the extractor.
+   */
+  public async close() {
+    this.abortController.abort();
+    this.frameBuffer.forEach(frame => frame.close());
+    try {
+      if (this.decoder.state === 'configured') {
+        await this.decoder.flush();
+        this.decoder.close();
+      }
+    } catch (e) {
+      // Ignore
+    }
+  }
+
+  public getFramesProcessed() {
+    return this.framesProcessed;
+  }
+
+  public getStartTime() {
+    return this.startTime;
+  }
+}